Initial commit: notifications, monitoring, on-call and incidents
One binary and one SQLite file. Topics with ntfy-style publish and subscribe, HTTP/TCP/DNS/TLS/heartbeat monitors with a public status page, schedules and escalation policies, and incidents with a timeline and post-mortem. The UI follows the Lightwell design language. Builds against an unreleased Reef (secretbox) through a local replace line. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
VeillaxandClaude Sonnet 5.5 committed
commit
0c5a57556a
22 files changed
+4953
No files matched your search
@@ -0,0 +1,860 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Handler returns the HTTP handler for the whole app.
|
||||
func (s *Server) Handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// pages
|
||||
mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) { servePage(w, indexHTML) })
|
||||
mux.HandleFunc("GET /status", func(w http.ResponseWriter, r *http.Request) { servePage(w, statusHTML) })
|
||||
mux.HandleFunc("GET /status.json", s.statusJSON)
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "ok") })
|
||||
mux.HandleFunc("GET /metrics", s.metrics)
|
||||
mux.HandleFunc("GET /api/v1/hb/{token}", s.heartbeat)
|
||||
mux.HandleFunc("POST /api/v1/hb/{token}", s.heartbeat)
|
||||
|
||||
// session
|
||||
mux.HandleFunc("GET /api/v1/state", s.getState)
|
||||
mux.HandleFunc("POST /api/v1/setup", s.setup)
|
||||
mux.HandleFunc("POST /api/v1/login", s.apiLogin)
|
||||
mux.HandleFunc("POST /api/v1/logout", func(w http.ResponseWriter, r *http.Request) { s.logout(w, r); writeJSON(w, 200, ok) })
|
||||
|
||||
// topics, ntfy-style
|
||||
mux.HandleFunc("POST /{topic}", s.auth(false, s.publishHTTP))
|
||||
mux.HandleFunc("PUT /{topic}", s.auth(false, s.publishHTTP))
|
||||
mux.HandleFunc("GET /{topic}/sse", s.auth(false, s.subscribeHTTP))
|
||||
mux.HandleFunc("GET /{topic}/json", s.auth(false, s.subscribeHTTP))
|
||||
mux.HandleFunc("GET /api/v1/topics/{topic}/messages", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
|
||||
writeJSON(w, 200, s.MessagesSince(r.PathValue("topic"), since, 200))
|
||||
}))
|
||||
|
||||
// people
|
||||
mux.HandleFunc("GET /api/v1/users", s.auth(false, func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, s.listUsers()) }))
|
||||
mux.HandleFunc("POST /api/v1/users", s.auth(true, s.createUserHTTP))
|
||||
mux.HandleFunc("DELETE /api/v1/users/{id}", s.auth(true, s.deleteUser))
|
||||
mux.HandleFunc("POST /api/v1/users/{id}/password", s.auth(false, s.passwordHTTP))
|
||||
mux.HandleFunc("GET /api/v1/apikeys", s.auth(false, s.listKeys))
|
||||
mux.HandleFunc("POST /api/v1/apikeys", s.auth(false, s.createKeyHTTP))
|
||||
mux.HandleFunc("DELETE /api/v1/apikeys/{id}", s.auth(false, s.deleteKey))
|
||||
|
||||
// configuration documents
|
||||
s.crud(mux, "channels", "channel", true, func() any { return &Channel{} }, func(v any, id string) error {
|
||||
ch := v.(*Channel)
|
||||
ch.ID = id
|
||||
if ch.Target == redactedTarget { // an edit that left the hidden secret alone keeps the stored one
|
||||
old, err := getOf[Channel](s.st, "channel", id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ch.Target = s.st.open(old.Target)
|
||||
}
|
||||
if err := validateChannel(ch); err != nil {
|
||||
return err
|
||||
}
|
||||
ch.Target = s.st.seal(ch.Target)
|
||||
return nil
|
||||
}, redactChannel)
|
||||
s.crud(mux, "monitors", "monitor", true, func() any { return &Monitor{} }, func(v any, id string) error {
|
||||
m := v.(*Monitor)
|
||||
m.ID = id
|
||||
return validateMonitor(m)
|
||||
}, nil)
|
||||
s.crud(mux, "schedules", "schedule", true, func() any { return &Schedule{} }, func(v any, id string) error {
|
||||
sc := v.(*Schedule)
|
||||
sc.ID = id
|
||||
if strings.TrimSpace(sc.Name) == "" {
|
||||
return errors.New("give the schedule a name")
|
||||
}
|
||||
if sc.RotationHours < 1 {
|
||||
sc.RotationHours = 24 * 7
|
||||
}
|
||||
if sc.Start == 0 {
|
||||
sc.Start = s.now().Unix()
|
||||
}
|
||||
for _, o := range sc.Overrides {
|
||||
if o.To <= o.From {
|
||||
return errors.New("an override must end after it starts")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, nil)
|
||||
s.crud(mux, "policies", "policy", true, func() any { return &Policy{} }, func(v any, id string) error {
|
||||
p := v.(*Policy)
|
||||
p.ID = id
|
||||
if strings.TrimSpace(p.Name) == "" {
|
||||
return errors.New("give the policy a name")
|
||||
}
|
||||
if len(p.Steps) == 0 {
|
||||
return errors.New("a policy needs at least one step")
|
||||
}
|
||||
return nil
|
||||
}, nil)
|
||||
mux.HandleFunc("POST /api/v1/channels/{id}/test", s.auth(true, s.testChannel))
|
||||
mux.HandleFunc("GET /api/v1/oncall", s.auth(false, s.onCallNow))
|
||||
|
||||
// monitors
|
||||
mux.HandleFunc("GET /api/v1/monitors/{id}/checks", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 200, s.recentChecks(r.PathValue("id"), 100))
|
||||
}))
|
||||
mux.HandleFunc("GET /api/v1/monitor-states", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
out := map[string]MonitorState{}
|
||||
mons, _ := listOf[Monitor](s.st, "monitor")
|
||||
for _, m := range mons {
|
||||
out[m.ID] = s.monitorState(m.ID)
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}))
|
||||
|
||||
// alerts
|
||||
mux.HandleFunc("GET /api/v1/alerts", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
writeJSON(w, 200, s.listAlerts(r.URL.Query().Get("status"), limit))
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/alerts", s.auth(false, s.inboundAlert))
|
||||
mux.HandleFunc("POST /api/v1/integrations/alertmanager", s.auth(false, s.alertmanager))
|
||||
mux.HandleFunc("GET /api/v1/alerts/{id}", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
a, err := s.getAlert(r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
a.Timeline = s.st.events("alert", a.ID)
|
||||
writeJSON(w, 200, a)
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/alerts/{id}/ack", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
fail2(w, s.AckAlert(r.PathValue("id"), userFrom(r.Context()).Name))
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/alerts/{id}/resolve", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
fail2(w, s.ResolveAlert(r.PathValue("id"), userFrom(r.Context()).Name))
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/alerts/{id}/snooze", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Minutes int `json:"minutes"`
|
||||
}
|
||||
decode(r, &in)
|
||||
if in.Minutes < 1 || in.Minutes > 24*60 {
|
||||
in.Minutes = 30
|
||||
}
|
||||
fail2(w, s.SnoozeAlert(r.PathValue("id"), userFrom(r.Context()).Name, time.Duration(in.Minutes)*time.Minute))
|
||||
}))
|
||||
|
||||
// incidents
|
||||
mux.HandleFunc("GET /api/v1/incidents", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
list, err := listOf[Incident](s.st, "incident")
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
sortIncidents(list) // ids are random, so order by time
|
||||
writeJSON(w, 200, list)
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/incidents", s.auth(false, s.declareHTTP))
|
||||
mux.HandleFunc("GET /api/v1/incidents/{id}", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
in, err := getOf[Incident](s.st, "incident", r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
in.Timeline = s.st.events("incident", in.ID)
|
||||
writeJSON(w, 200, in)
|
||||
}))
|
||||
mux.HandleFunc("PATCH /api/v1/incidents/{id}", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
var p IncidentPatch
|
||||
if !decode(r, &p) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
in, err := s.UpdateIncident(r.PathValue("id"), userFrom(r.Context()).Name, p)
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, in)
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/incidents/{id}/comments", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
decode(r, &in)
|
||||
fail2(w, s.CommentIncident(r.PathValue("id"), userFrom(r.Context()).Name, in.Text))
|
||||
}))
|
||||
mux.HandleFunc("POST /api/v1/incidents/{id}/postmortem-template", s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
in, err := getOf[Incident](s.st, "incident", r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]string{"template": s.postmortemTemplate(in)})
|
||||
}))
|
||||
mux.HandleFunc("DELETE /api/v1/incidents/{id}", s.auth(true, func(w http.ResponseWriter, r *http.Request) {
|
||||
fail2(w, s.st.del("incident", r.PathValue("id")))
|
||||
}))
|
||||
|
||||
return secureHeaders(mux)
|
||||
}
|
||||
|
||||
var ok = map[string]bool{"ok": true}
|
||||
|
||||
func servePage(w http.ResponseWriter, page []byte) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.Write(page)
|
||||
}
|
||||
|
||||
func secureHeaders(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h := w.Header()
|
||||
h.Set("X-Content-Type-Options", "nosniff")
|
||||
h.Set("Referrer-Policy", "no-referrer")
|
||||
h.Set("X-Frame-Options", "DENY")
|
||||
h.Set("Content-Security-Policy", "default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func writeJSON(w http.ResponseWriter, code int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
// bad answers 400. Messages are capitalized here, in one place, so errors read as sentences.
|
||||
func bad(w http.ResponseWriter, msg string) {
|
||||
if msg != "" {
|
||||
r, n := utf8.DecodeRuneInString(msg)
|
||||
msg = string(unicode.ToUpper(r)) + msg[n:]
|
||||
}
|
||||
writeJSON(w, 400, map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
func fail(w http.ResponseWriter, err error) {
|
||||
if errors.Is(err, errNotFound) {
|
||||
writeJSON(w, 404, map[string]string{"error": "not found"})
|
||||
return
|
||||
}
|
||||
bad(w, err.Error())
|
||||
}
|
||||
|
||||
// fail2 answers with an error, or with ok when err is nil.
|
||||
func fail2(w http.ResponseWriter, err error) {
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, ok)
|
||||
}
|
||||
|
||||
func decode(r *http.Request, v any) bool {
|
||||
return json.NewDecoder(http.MaxBytesReader(nil, r.Body, 1<<20)).Decode(v) == nil
|
||||
}
|
||||
|
||||
// auth wraps a handler: it needs a signed-in user (admin if adminOnly). Requests
|
||||
// that change things with a cookie must be JSON, which a cross-site form can't send.
|
||||
func (s *Server) auth(adminOnly bool, h http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
u := s.authenticate(r)
|
||||
if u == nil {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="sandbar"`)
|
||||
writeJSON(w, 401, map[string]string{"error": "sign in or send an API key"})
|
||||
return
|
||||
}
|
||||
if adminOnly && u.Role != "admin" {
|
||||
writeJSON(w, 403, map[string]string{"error": "administrators only"})
|
||||
return
|
||||
}
|
||||
_, noCookie := r.Cookie(sessionCookie)
|
||||
if noCookie == nil && r.Method != "GET" && strings.HasPrefix(r.URL.Path, "/api/") &&
|
||||
!strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
bad(w, "send application/json")
|
||||
return
|
||||
}
|
||||
h(w, r.WithContext(context.WithValue(r.Context(), ctxKey{}, u)))
|
||||
}
|
||||
}
|
||||
|
||||
// crud registers list/create/update/delete for a kind of configuration document.
|
||||
func (s *Server) crud(mux *http.ServeMux, path, kind string, adminWrite bool, newItem func() any, prepare func(v any, id string) error, redact func(any) any) {
|
||||
base := "/api/v1/" + path
|
||||
read := func(v any) any {
|
||||
if redact != nil {
|
||||
return redact(v)
|
||||
}
|
||||
return v
|
||||
}
|
||||
mux.HandleFunc("GET "+base, s.auth(false, func(w http.ResponseWriter, r *http.Request) {
|
||||
var out []any
|
||||
s.st.list(kind, func(raw []byte) error {
|
||||
v := newItem()
|
||||
if err := json.Unmarshal(raw, v); err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, read(v))
|
||||
return nil
|
||||
})
|
||||
if out == nil {
|
||||
out = []any{}
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}))
|
||||
save := func(w http.ResponseWriter, r *http.Request, id string) {
|
||||
v := newItem()
|
||||
if !decode(r, v) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
if err := prepare(v, id); err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
if err := s.st.put(kind, id, v); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, read(v))
|
||||
}
|
||||
mux.HandleFunc("POST "+base, s.auth(adminWrite, func(w http.ResponseWriter, r *http.Request) { save(w, r, newID()) }))
|
||||
mux.HandleFunc("PUT "+base+"/{id}", s.auth(adminWrite, func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if err := s.st.get(kind, id, newItem()); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
// A channel's secret target is never sent to the browser; an unchanged form sends the placeholder.
|
||||
save(w, r, id)
|
||||
}))
|
||||
mux.HandleFunc("DELETE "+base+"/{id}", s.auth(adminWrite, func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if kind == "monitor" {
|
||||
s.deleteMonitorData(id)
|
||||
}
|
||||
fail2(w, s.st.del(kind, id))
|
||||
}))
|
||||
}
|
||||
|
||||
func redactChannel(v any) any {
|
||||
c := *(v.(*Channel))
|
||||
c.Target = redactedTarget // webhook URLs and addresses are secrets; re-enter to change
|
||||
return c
|
||||
}
|
||||
|
||||
const redactedTarget = "••••••••"
|
||||
|
||||
// ---------- session ----------
|
||||
|
||||
func (s *Server) getState(w http.ResponseWriter, r *http.Request) {
|
||||
out := map[string]any{"needsSetup": s.countUsers() == 0, "user": nil, "url": s.opts.URL}
|
||||
if u := s.authenticate(r); u != nil {
|
||||
out["user"] = u
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) setup(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct{ Name, Email, Password string }
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
if s.countUsers() > 0 {
|
||||
writeJSON(w, 403, map[string]string{"error": "already set up"})
|
||||
return
|
||||
}
|
||||
if _, err := s.createUser(in.Name, in.Email, in.Password, "admin", ""); err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
u, err := s.login(w, r, in.Email, in.Password)
|
||||
if err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, u)
|
||||
}
|
||||
|
||||
func (s *Server) apiLogin(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct{ Email, Password string }
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
u, err := s.login(w, r, in.Email, in.Password)
|
||||
if err != nil {
|
||||
code := 401
|
||||
if errors.Is(err, errThrottled) {
|
||||
code = 429
|
||||
}
|
||||
writeJSON(w, code, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, u)
|
||||
}
|
||||
|
||||
// ---------- people ----------
|
||||
|
||||
func (s *Server) createUserHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct{ Name, Email, Password, Role, Phone string }
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
if in.Role == "" {
|
||||
in.Role = "member"
|
||||
}
|
||||
u, err := s.createUser(in.Name, in.Email, in.Password, in.Role, in.Phone)
|
||||
if err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, u)
|
||||
}
|
||||
|
||||
func (s *Server) deleteUser(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if id == userFrom(r.Context()).ID {
|
||||
bad(w, "you can't delete yourself")
|
||||
return
|
||||
}
|
||||
res, err := s.st.db.Exec(`DELETE FROM users WHERE id=?`, id)
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
fail(w, errNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, ok)
|
||||
}
|
||||
|
||||
func (s *Server) passwordHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
me := userFrom(r.Context())
|
||||
id := r.PathValue("id")
|
||||
if id != me.ID && me.Role != "admin" {
|
||||
writeJSON(w, 403, map[string]string{"error": "you can only change your own password"})
|
||||
return
|
||||
}
|
||||
var in struct{ Password string }
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
fail2(w, s.setPassword(id, in.Password))
|
||||
}
|
||||
|
||||
func (s *Server) listKeys(w http.ResponseWriter, r *http.Request) {
|
||||
out := []map[string]any{}
|
||||
rows, err := s.st.db.Query(`SELECT id,name,created FROM apikeys WHERE user_id=? ORDER BY created`, userFrom(r.Context()).ID)
|
||||
if err == nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id, name string
|
||||
var c int64
|
||||
if rows.Scan(&id, &name, &c) == nil {
|
||||
out = append(out, map[string]any{"id": id, "name": name, "created": c})
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) createKeyHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct{ Name string }
|
||||
decode(r, &in)
|
||||
id, tok, err := s.createAPIKey(userFrom(r.Context()).ID, in.Name)
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]string{"id": id, "token": tok}) // shown once
|
||||
}
|
||||
|
||||
func (s *Server) deleteKey(w http.ResponseWriter, r *http.Request) {
|
||||
s.st.db.Exec(`DELETE FROM apikeys WHERE id=? AND user_id=?`, r.PathValue("id"), userFrom(r.Context()).ID)
|
||||
writeJSON(w, 200, ok)
|
||||
}
|
||||
|
||||
// ---------- topics ----------
|
||||
|
||||
func (s *Server) publishHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
topic := r.PathValue("topic")
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
|
||||
if err != nil {
|
||||
bad(w, "message too large (limit 64 KB)")
|
||||
return
|
||||
}
|
||||
h := func(names ...string) string {
|
||||
for _, n := range names {
|
||||
if v := r.Header.Get(n); v != "" {
|
||||
return v
|
||||
}
|
||||
if v := r.URL.Query().Get(strings.ToLower(n)); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
m, err := s.Publish(topic, h("Title", "X-Title", "t"), string(body), parsePriority(h("Priority", "X-Priority", "p")), parseTags(h("Tags", "X-Tags", "ta")))
|
||||
if err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, m)
|
||||
}
|
||||
|
||||
// subscribeHTTP streams a topic as server-sent events (/sse) or JSON lines (/json).
|
||||
// ?since=<unix seconds>|all replays stored messages first.
|
||||
func (s *Server) subscribeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
topic := r.PathValue("topic")
|
||||
if !validTopic(topic) {
|
||||
bad(w, "invalid topic")
|
||||
return
|
||||
}
|
||||
fl, canFlush := w.(http.Flusher)
|
||||
if !canFlush {
|
||||
http.Error(w, "streaming not supported", 500)
|
||||
return
|
||||
}
|
||||
sse := strings.HasSuffix(r.URL.Path, "/sse")
|
||||
if sse {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "application/x-ndjson")
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("X-Accel-Buffering", "no")
|
||||
bw := bufio.NewWriter(w)
|
||||
write := func(m Message) bool {
|
||||
b, _ := json.Marshal(m)
|
||||
var err error
|
||||
if sse {
|
||||
_, err = fmt.Fprintf(bw, "id: %s\ndata: %s\n\n", m.ID, b)
|
||||
} else {
|
||||
_, err = fmt.Fprintf(bw, "%s\n", b)
|
||||
}
|
||||
bw.Flush()
|
||||
fl.Flush()
|
||||
return err == nil
|
||||
}
|
||||
// Subscribe before replaying so nothing published in between is missed; duplicates are skipped by id.
|
||||
ch, unsub := s.hub.subscribe(topic)
|
||||
defer unsub()
|
||||
seen := map[string]bool{}
|
||||
if q := r.URL.Query().Get("since"); q != "" {
|
||||
since := int64(0)
|
||||
if q != "all" {
|
||||
since, _ = strconv.ParseInt(q, 10, 64)
|
||||
}
|
||||
for _, m := range s.MessagesSince(topic, since, 500) {
|
||||
seen[m.ID] = true
|
||||
if !write(m) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if sse {
|
||||
fmt.Fprint(bw, ": connected\n\n")
|
||||
bw.Flush()
|
||||
fl.Flush()
|
||||
}
|
||||
keepalive := time.NewTicker(30 * time.Second)
|
||||
defer keepalive.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-s.stop:
|
||||
return
|
||||
case m := <-ch:
|
||||
if seen[m.ID] {
|
||||
continue
|
||||
}
|
||||
if !write(m) {
|
||||
return
|
||||
}
|
||||
case <-keepalive.C:
|
||||
if sse {
|
||||
fmt.Fprint(bw, ": keepalive\n\n")
|
||||
} else {
|
||||
fmt.Fprint(bw, "\n")
|
||||
}
|
||||
bw.Flush()
|
||||
fl.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- alerts in ----------
|
||||
|
||||
func (s *Server) inboundAlert(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Key string `json:"key"`
|
||||
Summary string `json:"summary"`
|
||||
Detail string `json:"detail"`
|
||||
Severity string `json:"severity"`
|
||||
Policy string `json:"policy"`
|
||||
Status string `json:"status"` // firing (default) or resolved
|
||||
Source string `json:"source"`
|
||||
}
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
if in.Key == "" {
|
||||
in.Key = in.Summary
|
||||
}
|
||||
if in.Key == "" {
|
||||
bad(w, "send a key or a summary")
|
||||
return
|
||||
}
|
||||
if in.Source == "" {
|
||||
in.Source = "api"
|
||||
}
|
||||
in.Key = "ext:" + in.Key
|
||||
if in.Status == "resolved" {
|
||||
s.ResolveAlertKey(in.Key, in.Source)
|
||||
writeJSON(w, 200, ok)
|
||||
return
|
||||
}
|
||||
if in.Summary == "" {
|
||||
in.Summary = in.Key
|
||||
}
|
||||
if in.Policy != "" {
|
||||
if _, err := getOf[Policy](s.st, "policy", in.Policy); err != nil {
|
||||
bad(w, "unknown escalation policy")
|
||||
return
|
||||
}
|
||||
}
|
||||
a, isNew, err := s.RaiseAlert(in.Key, in.Source, in.Summary, in.Detail, in.Severity, in.Policy)
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"id": a.ID, "new": isNew})
|
||||
}
|
||||
|
||||
// alertmanager accepts Prometheus Alertmanager's webhook (also Grafana's, which uses the same shape).
|
||||
// ?policy=<id> sets the escalation policy.
|
||||
func (s *Server) alertmanager(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Alerts []struct {
|
||||
Status string `json:"status"`
|
||||
Labels map[string]string `json:"labels"`
|
||||
Annotations map[string]string `json:"annotations"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
} `json:"alerts"`
|
||||
}
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
policy := r.URL.Query().Get("policy")
|
||||
if policy != "" {
|
||||
if _, err := getOf[Policy](s.st, "policy", policy); err != nil {
|
||||
bad(w, "unknown escalation policy")
|
||||
return
|
||||
}
|
||||
}
|
||||
n := 0
|
||||
for _, a := range in.Alerts {
|
||||
key := a.Fingerprint
|
||||
if key == "" {
|
||||
key = a.Labels["alertname"] + "/" + a.Labels["instance"]
|
||||
}
|
||||
key = "am:" + key
|
||||
if a.Status == "resolved" {
|
||||
s.ResolveAlertKey(key, "alertmanager")
|
||||
continue
|
||||
}
|
||||
sev := map[string]string{"critical": "critical", "warning": "warning", "info": "info"}[a.Labels["severity"]]
|
||||
summary := firstNonEmpty(a.Annotations["summary"], a.Labels["alertname"], key)
|
||||
if _, _, err := s.RaiseAlert(key, "alertmanager", summary, a.Annotations["description"], sev, policy); err == nil {
|
||||
n++
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, map[string]int{"processed": n})
|
||||
}
|
||||
|
||||
// ---------- misc ----------
|
||||
|
||||
func (s *Server) declareHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
var in struct {
|
||||
Title string `json:"title"`
|
||||
Severity string `json:"severity"`
|
||||
Summary string `json:"summary"`
|
||||
Lead string `json:"lead"`
|
||||
AlertIDs []string `json:"alertIds"`
|
||||
}
|
||||
if !decode(r, &in) {
|
||||
bad(w, "invalid request")
|
||||
return
|
||||
}
|
||||
me := userFrom(r.Context())
|
||||
if in.Lead == "" {
|
||||
in.Lead = me.ID
|
||||
}
|
||||
inc, err := s.DeclareIncident(in.Title, in.Severity, in.Summary, in.Lead, me.Name, in.AlertIDs)
|
||||
if err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, inc)
|
||||
}
|
||||
|
||||
func (s *Server) testChannel(w http.ResponseWriter, r *http.Request) {
|
||||
ch, err := getOf[Channel](s.st, "channel", r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
if err := s.sendChannel(*ch, Note{Status: "test", Kind: "alert", Title: "test", Time: s.now().Unix()}); err != nil {
|
||||
bad(w, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, ok)
|
||||
}
|
||||
|
||||
func (s *Server) onCallNow(w http.ResponseWriter, r *http.Request) {
|
||||
type row struct {
|
||||
Schedule string `json:"schedule"`
|
||||
ID string `json:"id"`
|
||||
UserID string `json:"userId"`
|
||||
User string `json:"user"`
|
||||
Next string `json:"next"` // who is on call after the current handover
|
||||
}
|
||||
out := []row{}
|
||||
scs, _ := listOf[Schedule](s.st, "schedule")
|
||||
now := s.now()
|
||||
for _, sc := range scs {
|
||||
sc := sc
|
||||
uid := sc.OnCall(now)
|
||||
nx := sc.OnCall(now.Add(time.Duration(sc.RotationHours) * time.Hour))
|
||||
out = append(out, row{sc.Name, sc.ID, uid, s.userName(uid), s.userName(nx)})
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) heartbeat(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.Heartbeat(r.PathValue("token")) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
io.WriteString(w, "ok")
|
||||
}
|
||||
|
||||
// statusJSON feeds the public status page: only monitors and incidents marked public.
|
||||
func (s *Server) statusJSON(w http.ResponseWriter, r *http.Request) {
|
||||
type mon struct {
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Uptime float64 `json:"uptime"`
|
||||
Since int64 `json:"since"`
|
||||
}
|
||||
type inc struct {
|
||||
Title string `json:"title"`
|
||||
Status string `json:"status"`
|
||||
Severity string `json:"severity"`
|
||||
Created int64 `json:"created"`
|
||||
Resolved int64 `json:"resolved"`
|
||||
Updates []Event `json:"updates"`
|
||||
}
|
||||
out := struct {
|
||||
Monitors []mon `json:"monitors"`
|
||||
Incidents []inc `json:"incidents"`
|
||||
Overall string `json:"overall"`
|
||||
}{[]mon{}, []inc{}, "operational"}
|
||||
mons, _ := listOf[Monitor](s.st, "monitor")
|
||||
for _, m := range mons {
|
||||
if !m.Public || !m.Enabled {
|
||||
continue
|
||||
}
|
||||
st := s.monitorState(m.ID)
|
||||
out.Monitors = append(out.Monitors, mon{m.Name, st.Status, st.Uptime30, st.Since})
|
||||
if st.Status == "down" {
|
||||
out.Overall = "outage"
|
||||
}
|
||||
}
|
||||
incs, _ := listOf[Incident](s.st, "incident")
|
||||
sortIncidents(incs)
|
||||
cut := s.now().AddDate(0, 0, -14).Unix()
|
||||
for _, in := range incs {
|
||||
if !in.Public || (in.Status == "resolved" && in.Resolved < cut) {
|
||||
continue
|
||||
}
|
||||
var ups []Event
|
||||
for _, e := range s.st.events("incident", in.ID) {
|
||||
if e.Kind == "update" || e.Kind == "declared" {
|
||||
e.Actor = "" // staff names stay off the public page
|
||||
ups = append(ups, e)
|
||||
}
|
||||
}
|
||||
out.Incidents = append(out.Incidents, inc{in.Title, in.Status, in.Severity, in.Created, in.Resolved, ups})
|
||||
if in.Status != "resolved" && out.Overall == "operational" {
|
||||
out.Overall = "degraded"
|
||||
}
|
||||
}
|
||||
w.Header().Set("Cache-Control", "max-age=15")
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
|
||||
func sortIncidents(list []Incident) {
|
||||
for i := 1; i < len(list); i++ { // newest first; lists are short
|
||||
for j := i; j > 0 && list[j].Created > list[j-1].Created; j-- {
|
||||
list[j], list[j-1] = list[j-1], list[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) metrics(w http.ResponseWriter, r *http.Request) {
|
||||
if s.opts.MetricsToken == "" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if h := r.Header.Get("Authorization"); h != "Bearer "+s.opts.MetricsToken {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="metrics"`)
|
||||
http.Error(w, "unauthorized", 401)
|
||||
return
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("# TYPE sandbar_monitor_up gauge\n")
|
||||
mons, _ := listOf[Monitor](s.st, "monitor")
|
||||
for _, m := range mons {
|
||||
v := 0
|
||||
if s.monitorState(m.ID).Status == "up" {
|
||||
v = 1
|
||||
}
|
||||
fmt.Fprintf(&b, "sandbar_monitor_up{monitor=%q} %d\n", m.Name, v)
|
||||
}
|
||||
var firing, acked, open int
|
||||
s.st.db.QueryRow(`SELECT COUNT(*) FROM alerts WHERE status='firing'`).Scan(&firing)
|
||||
s.st.db.QueryRow(`SELECT COUNT(*) FROM alerts WHERE status='acked'`).Scan(&acked)
|
||||
incs, _ := listOf[Incident](s.st, "incident")
|
||||
for _, in := range incs {
|
||||
if in.Status != "resolved" {
|
||||
open++
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "# TYPE sandbar_alerts gauge\nsandbar_alerts{status=\"firing\"} %d\nsandbar_alerts{status=\"acked\"} %d\n", firing, acked)
|
||||
fmt.Fprintf(&b, "# TYPE sandbar_incidents_open gauge\nsandbar_incidents_open %d\n", open)
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
|
||||
io.WriteString(w, b.String())
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Email string `json:"email"`
|
||||
Role string `json:"role"` // admin or member
|
||||
Phone string `json:"phone"`
|
||||
Created int64 `json:"created"`
|
||||
Topic string `json:"topic"` // the personal topic their phone subscribes to
|
||||
hash string
|
||||
}
|
||||
|
||||
const (
|
||||
sessionCookie = "sandbar_session"
|
||||
sessionTTL = 14 * 24 * time.Hour
|
||||
)
|
||||
|
||||
type ctxKey struct{}
|
||||
|
||||
func (s *Server) countUsers() int {
|
||||
var n int
|
||||
s.st.db.QueryRow(`SELECT COUNT(*) FROM users`).Scan(&n)
|
||||
return n
|
||||
}
|
||||
|
||||
func scanUser(r scanner) (*User, error) {
|
||||
var u User
|
||||
err := r.Scan(&u.ID, &u.Name, &u.Email, &u.hash, &u.Role, &u.Phone, &u.Created)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, errNotFound
|
||||
}
|
||||
u.Topic = userTopic(u.ID)
|
||||
return &u, err
|
||||
}
|
||||
|
||||
const userCols = `id,name,email,hash,role,phone,created`
|
||||
|
||||
func (s *Server) userByID(id string) (*User, error) {
|
||||
return scanUser(s.st.db.QueryRow(`SELECT `+userCols+` FROM users WHERE id=?`, id))
|
||||
}
|
||||
|
||||
func (s *Server) userByEmail(email string) (*User, error) {
|
||||
return scanUser(s.st.db.QueryRow(`SELECT `+userCols+` FROM users WHERE email=?`, strings.TrimSpace(email)))
|
||||
}
|
||||
|
||||
func (s *Server) listUsers() []User {
|
||||
out := []User{}
|
||||
rows, err := s.st.db.Query(`SELECT ` + userCols + ` FROM users ORDER BY name`)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
if u, err := scanUser(rows); err == nil {
|
||||
out = append(out, *u)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func validPassword(p string) error {
|
||||
if len(p) < 10 {
|
||||
return errors.New("use a password of at least 10 characters")
|
||||
}
|
||||
if len(p) > 72 {
|
||||
return errors.New("passwords are limited to 72 characters")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) createUser(name, email, password, role, phone string) (*User, error) {
|
||||
name, email = strings.TrimSpace(name), strings.TrimSpace(email)
|
||||
if name == "" || !strings.Contains(email, "@") || strings.ContainsAny(email, " \r\n<>") {
|
||||
return nil, errors.New("a user needs a name and an email address")
|
||||
}
|
||||
if role != "admin" && role != "member" {
|
||||
return nil, errors.New("role must be admin or member")
|
||||
}
|
||||
if err := validPassword(password); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u := &User{ID: newID(), Name: name, Email: email, Role: role, Phone: strings.TrimSpace(phone), Created: s.now().Unix()}
|
||||
u.Topic = userTopic(u.ID)
|
||||
_, err = s.st.db.Exec(`INSERT INTO users(`+userCols+`) VALUES(?,?,?,?,?,?,?)`, u.ID, u.Name, u.Email, string(h), u.Role, u.Phone, u.Created)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "UNIQUE") {
|
||||
return nil, errors.New("that email address is already registered")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (s *Server) setPassword(id, password string) error {
|
||||
if err := validPassword(password); err != nil {
|
||||
return err
|
||||
}
|
||||
h, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.st.db.Exec(`UPDATE users SET hash=? WHERE id=?`, string(h), id)
|
||||
if err == nil {
|
||||
s.st.db.Exec(`DELETE FROM sessions WHERE user_id=?`, id) // a password change signs everyone out
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// throttled limits password guessing per client address: 10 tries per 10 minutes.
|
||||
func (s *Server) throttled(ip string) bool {
|
||||
s.loginMu.Lock()
|
||||
defer s.loginMu.Unlock()
|
||||
cut := s.now().Add(-10 * time.Minute)
|
||||
var keep []time.Time
|
||||
for _, t := range s.loginTry[ip] {
|
||||
if t.After(cut) {
|
||||
keep = append(keep, t)
|
||||
}
|
||||
}
|
||||
s.loginTry[ip] = keep
|
||||
return len(keep) >= 10
|
||||
}
|
||||
|
||||
func (s *Server) noteFailure(ip string) {
|
||||
s.loginMu.Lock()
|
||||
s.loginTry[ip] = append(s.loginTry[ip], s.now())
|
||||
s.loginMu.Unlock()
|
||||
}
|
||||
|
||||
func clientIP(r *http.Request) string {
|
||||
h, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
return r.RemoteAddr
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
var errThrottled = errors.New("too many attempts, try again in a few minutes")
|
||||
|
||||
// login checks credentials and starts a session.
|
||||
func (s *Server) login(w http.ResponseWriter, r *http.Request, email, password string) (*User, error) {
|
||||
ip := clientIP(r)
|
||||
if s.throttled(ip) {
|
||||
return nil, errThrottled
|
||||
}
|
||||
u, err := s.userByEmail(email)
|
||||
hash := "$2a$10$7EqJtq98hPqEX7fNZaFWoOhi5BWYKTaYBwxwEGLMzRVr4qhCXWM9S" // so a wrong email costs the same time as a wrong password
|
||||
if err == nil {
|
||||
hash = u.hash
|
||||
}
|
||||
if bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) != nil || err != nil {
|
||||
s.noteFailure(ip)
|
||||
return nil, errors.New("wrong email or password")
|
||||
}
|
||||
tok := newToken()
|
||||
exp := s.now().Add(sessionTTL)
|
||||
if _, err := s.st.db.Exec(`INSERT INTO sessions(hash,user_id,expires) VALUES(?,?,?)`, hashToken(tok), u.ID, exp.Unix()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{Name: sessionCookie, Value: tok, Path: "/", Expires: exp, HttpOnly: true,
|
||||
SameSite: http.SameSiteStrictMode, Secure: strings.HasPrefix(s.opts.URL, "https://")})
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (s *Server) logout(w http.ResponseWriter, r *http.Request) {
|
||||
if c, err := r.Cookie(sessionCookie); err == nil {
|
||||
s.st.db.Exec(`DELETE FROM sessions WHERE hash=?`, hashToken(c.Value))
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{Name: sessionCookie, Value: "", Path: "/", MaxAge: -1, HttpOnly: true})
|
||||
}
|
||||
|
||||
// authenticate finds who is calling: a session cookie, or an API key given as a
|
||||
// bearer token or as the password of HTTP basic auth (ntfy clients do the latter).
|
||||
func (s *Server) authenticate(r *http.Request) *User {
|
||||
if c, err := r.Cookie(sessionCookie); err == nil {
|
||||
var uid string
|
||||
var exp int64
|
||||
if s.st.db.QueryRow(`SELECT user_id,expires FROM sessions WHERE hash=?`, hashToken(c.Value)).Scan(&uid, &exp) == nil && exp > s.now().Unix() {
|
||||
if u, err := s.userByID(uid); err == nil {
|
||||
return u
|
||||
}
|
||||
}
|
||||
}
|
||||
key := ""
|
||||
if h := r.Header.Get("Authorization"); strings.HasPrefix(h, "Bearer ") {
|
||||
key = strings.TrimSpace(h[7:])
|
||||
} else if _, pw, ok := r.BasicAuth(); ok {
|
||||
key = pw
|
||||
}
|
||||
if key == "" {
|
||||
return nil
|
||||
}
|
||||
var uid string
|
||||
if s.st.db.QueryRow(`SELECT user_id FROM apikeys WHERE hash=?`, hashToken(key)).Scan(&uid) == nil {
|
||||
if u, err := s.userByID(uid); err == nil {
|
||||
return u
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func userFrom(ctx context.Context) *User {
|
||||
u, _ := ctx.Value(ctxKey{}).(*User)
|
||||
return u
|
||||
}
|
||||
|
||||
func (s *Server) createAPIKey(userID, name string) (id, token string, err error) {
|
||||
token = "sb_" + newToken()
|
||||
id = newID()
|
||||
_, err = s.st.db.Exec(`INSERT INTO apikeys(id,name,hash,user_id,created) VALUES(?,?,?,?,?)`, id, strings.TrimSpace(name), hashToken(token), userID, s.now().Unix())
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Incidents are the human side: something is wrong and people are working on it.
|
||||
// An incident has a severity, a status that moves forward, a lead and a comms
|
||||
// owner, a timeline of everything that happened, and a post-mortem. Every change
|
||||
// is announced on the "incidents" topic and the broadcast channels.
|
||||
|
||||
type Incident struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Severity string `json:"severity"` // sev1 (worst) to sev3
|
||||
Status string `json:"status"` // investigating, identified, monitoring, resolved
|
||||
Lead string `json:"lead"` // user id
|
||||
Comms string `json:"comms"` // user id
|
||||
Summary string `json:"summary"`
|
||||
Postmortem string `json:"postmortem"`
|
||||
Public bool `json:"public"` // shown on the status page
|
||||
Created int64 `json:"created"`
|
||||
Resolved int64 `json:"resolved"`
|
||||
AlertIDs []string `json:"alertIds"`
|
||||
Timeline []Event `json:"timeline,omitempty"`
|
||||
}
|
||||
|
||||
var incidentStatuses = []string{"investigating", "identified", "monitoring", "resolved"}
|
||||
|
||||
func validStatus(s string) bool {
|
||||
for _, v := range incidentStatuses {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func validSeverity(s string) bool { return s == "sev1" || s == "sev2" || s == "sev3" }
|
||||
|
||||
func (s *Server) userName(id string) string {
|
||||
if u, err := s.userByID(id); err == nil {
|
||||
return u.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Server) announceIncident(in *Incident, status, text string) {
|
||||
s.broadcast(Note{Status: status, Kind: "incident", Title: in.Title, Message: fmt.Sprintf("[%s] %s: %s", strings.ToUpper(in.Severity), in.Status, text),
|
||||
Severity: map[string]string{"sev1": "critical", "sev2": "warning"}[in.Severity], Link: s.link("/#incidents"), Time: s.now().Unix()}, "incidents")
|
||||
}
|
||||
|
||||
// DeclareIncident opens an incident, optionally linking the alerts that caused it.
|
||||
func (s *Server) DeclareIncident(title, severity, summary, lead, actor string, alertIDs []string) (*Incident, error) {
|
||||
title = strings.TrimSpace(title)
|
||||
if title == "" {
|
||||
return nil, errors.New("give the incident a title")
|
||||
}
|
||||
if severity == "" {
|
||||
severity = "sev2"
|
||||
}
|
||||
if !validSeverity(severity) {
|
||||
return nil, errors.New("severity must be sev1, sev2 or sev3")
|
||||
}
|
||||
now := s.now()
|
||||
in := &Incident{ID: newID(), Title: title, Severity: severity, Status: "investigating", Lead: lead, Summary: summary, Created: now.Unix(), AlertIDs: []string{}}
|
||||
for _, id := range alertIDs {
|
||||
if a, err := s.getAlert(id); err == nil {
|
||||
in.AlertIDs = append(in.AlertIDs, a.ID)
|
||||
s.st.db.Exec(`UPDATE alerts SET incident_id=? WHERE id=?`, in.ID, a.ID)
|
||||
s.st.addEvent("alert", a.ID, "incident", actor, "linked to incident "+title, now)
|
||||
}
|
||||
}
|
||||
if err := s.st.put("incident", in.ID, in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.st.addEvent("incident", in.ID, "declared", actor, fmt.Sprintf("declared %s: %s", severity, title), now)
|
||||
if lead != "" {
|
||||
s.st.addEvent("incident", in.ID, "role", actor, "lead: "+s.userName(lead), now)
|
||||
}
|
||||
s.announceIncident(in, "declared", firstNonEmpty(summary, "incident declared"))
|
||||
return in, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(v ...string) string {
|
||||
for _, s := range v {
|
||||
if s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// IncidentPatch holds the fields an update may change; nil means unchanged.
|
||||
type IncidentPatch struct {
|
||||
Title *string `json:"title"`
|
||||
Severity *string `json:"severity"`
|
||||
Status *string `json:"status"`
|
||||
Lead *string `json:"lead"`
|
||||
Comms *string `json:"comms"`
|
||||
Summary *string `json:"summary"`
|
||||
Postmortem *string `json:"postmortem"`
|
||||
Public *bool `json:"public"`
|
||||
Note string `json:"note"` // text for the update announcement and timeline
|
||||
}
|
||||
|
||||
func (s *Server) UpdateIncident(id, actor string, p IncidentPatch) (*Incident, error) {
|
||||
in, err := getOf[Incident](s.st, "incident", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
var changes []string
|
||||
if p.Severity != nil && *p.Severity != in.Severity {
|
||||
if !validSeverity(*p.Severity) {
|
||||
return nil, errors.New("severity must be sev1, sev2 or sev3")
|
||||
}
|
||||
changes = append(changes, "severity "+in.Severity+" to "+*p.Severity)
|
||||
in.Severity = *p.Severity
|
||||
}
|
||||
if p.Status != nil && *p.Status != in.Status {
|
||||
if !validStatus(*p.Status) {
|
||||
return nil, errors.New("status must be investigating, identified, monitoring or resolved")
|
||||
}
|
||||
changes = append(changes, "status "+in.Status+" to "+*p.Status)
|
||||
in.Status = *p.Status
|
||||
if in.Status == "resolved" {
|
||||
in.Resolved = now.Unix()
|
||||
} else {
|
||||
in.Resolved = 0
|
||||
}
|
||||
}
|
||||
if p.Lead != nil && *p.Lead != in.Lead {
|
||||
in.Lead = *p.Lead
|
||||
changes = append(changes, "lead: "+s.userName(in.Lead))
|
||||
}
|
||||
if p.Comms != nil && *p.Comms != in.Comms {
|
||||
in.Comms = *p.Comms
|
||||
changes = append(changes, "comms: "+s.userName(in.Comms))
|
||||
}
|
||||
if p.Title != nil && strings.TrimSpace(*p.Title) != "" && *p.Title != in.Title {
|
||||
in.Title = strings.TrimSpace(*p.Title)
|
||||
changes = append(changes, "renamed")
|
||||
}
|
||||
if p.Summary != nil {
|
||||
in.Summary = *p.Summary
|
||||
}
|
||||
if p.Postmortem != nil {
|
||||
in.Postmortem = *p.Postmortem
|
||||
}
|
||||
if p.Public != nil {
|
||||
in.Public = *p.Public
|
||||
}
|
||||
if err := s.st.put("incident", in.ID, in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(changes) > 0 || p.Note != "" {
|
||||
text := strings.Join(changes, "; ")
|
||||
if p.Note != "" {
|
||||
text = strings.TrimSpace(text + ": " + p.Note)
|
||||
}
|
||||
s.st.addEvent("incident", in.ID, "update", actor, text, now)
|
||||
kind := "update"
|
||||
if in.Status == "resolved" {
|
||||
kind = "resolved"
|
||||
}
|
||||
s.announceIncident(in, kind, text)
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
func (s *Server) CommentIncident(id, actor, text string) error {
|
||||
if _, err := getOf[Incident](s.st, "incident", id); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return errors.New("write something")
|
||||
}
|
||||
s.st.addEvent("incident", id, "comment", actor, text, s.now())
|
||||
return nil
|
||||
}
|
||||
|
||||
// postmortemTemplate is a starting point, filled with what the timeline already knows.
|
||||
func (s *Server) postmortemTemplate(in *Incident) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "# Post-mortem: %s\n\n", in.Title)
|
||||
fmt.Fprintf(&b, "Severity: %s\nStarted: %s\n", in.Severity, time.Unix(in.Created, 0).UTC().Format(time.RFC1123))
|
||||
if in.Resolved > 0 {
|
||||
fmt.Fprintf(&b, "Resolved: %s (%s)\n", time.Unix(in.Resolved, 0).UTC().Format(time.RFC1123), time.Duration(in.Resolved-in.Created)*time.Second)
|
||||
}
|
||||
b.WriteString("\n## Summary\n\n\n## Impact\n\n\n## Root cause\n\n\n## Timeline\n\n")
|
||||
for _, e := range s.st.events("incident", in.ID) {
|
||||
fmt.Fprintf(&b, "- %s %s\n", time.Unix(e.Time, 0).UTC().Format("15:04:05"), e.Text)
|
||||
}
|
||||
b.WriteString("\n## What went well\n\n\n## What went badly\n\n\n## Action items\n\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.veillax.com/Veillax/reef/netguard"
|
||||
)
|
||||
|
||||
// Monitors check that things are up. A check runs every interval; the monitor
|
||||
// goes down after Retries failures in a row (so a blip is ignored) and raises an
|
||||
// alert, and recovers on the first success. Heartbeat monitors work the other
|
||||
// way round: your cron job or service calls /hb/<token>, and the monitor goes
|
||||
// down when the calls stop.
|
||||
|
||||
type Monitor struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"` // http, tcp, dns, tls, heartbeat
|
||||
Target string `json:"target"` // URL, host:port, hostname, or host[:port] for tls
|
||||
IntervalS int `json:"intervalS"`
|
||||
TimeoutS int `json:"timeoutS"`
|
||||
Retries int `json:"retries"` // failures in a row before it is down (default 2)
|
||||
Expect string `json:"expect"` // http: accepted status, e.g. "200-299" (default)
|
||||
Keyword string `json:"keyword"` // http: text the body must contain
|
||||
WarnDays int `json:"warnDays"` // tls: fail when the certificate expires within this many days (default 14)
|
||||
GraceS int `json:"graceS"` // heartbeat: extra time allowed past the interval
|
||||
Token string `json:"token"` // heartbeat: the secret in the ping URL
|
||||
PolicyID string `json:"policyId"` // escalation policy for the alert it raises
|
||||
Severity string `json:"severity"` // alert severity (default critical)
|
||||
Public bool `json:"public"` // shown on the status page
|
||||
AllowPrivate bool `json:"allowPrivate"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// MonitorState is what we know about a monitor right now.
|
||||
type MonitorState struct {
|
||||
MonitorID string `json:"monitorId"`
|
||||
Status string `json:"status"` // pending, up, down
|
||||
Since int64 `json:"since"`
|
||||
LastCheck int64 `json:"lastCheck"`
|
||||
LastMS int `json:"lastMs"`
|
||||
Fails int `json:"fails"`
|
||||
LastPing int64 `json:"lastPing"`
|
||||
Detail string `json:"detail"`
|
||||
Uptime30 float64 `json:"uptime30"` // percent of checks that passed in the retention window; -1 if none
|
||||
}
|
||||
|
||||
// interval is how often the monitor is checked. A heartbeat is evaluated often so
|
||||
// a missed one is noticed soon after its deadline, not up to a whole interval later.
|
||||
func (m *Monitor) interval() time.Duration {
|
||||
if m.Kind == "heartbeat" {
|
||||
return 15 * time.Second
|
||||
}
|
||||
if m.IntervalS < 10 {
|
||||
return time.Minute
|
||||
}
|
||||
return time.Duration(m.IntervalS) * time.Second
|
||||
}
|
||||
|
||||
func (m *Monitor) timeout() time.Duration {
|
||||
if m.TimeoutS <= 0 || m.TimeoutS > 120 {
|
||||
return 10 * time.Second
|
||||
}
|
||||
return time.Duration(m.TimeoutS) * time.Second
|
||||
}
|
||||
|
||||
func (m *Monitor) retries() int {
|
||||
if m.Retries < 1 {
|
||||
return 2
|
||||
}
|
||||
return m.Retries
|
||||
}
|
||||
|
||||
func validateMonitor(m *Monitor) error {
|
||||
m.Name, m.Target = strings.TrimSpace(m.Name), strings.TrimSpace(m.Target)
|
||||
if m.Name == "" {
|
||||
return errors.New("give the monitor a name")
|
||||
}
|
||||
switch m.Kind {
|
||||
case "http":
|
||||
u, err := url.Parse(m.Target)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return errors.New("an http monitor needs an http:// or https:// URL")
|
||||
}
|
||||
case "tcp":
|
||||
if _, _, err := net.SplitHostPort(m.Target); err != nil {
|
||||
return errors.New("a tcp monitor needs host:port")
|
||||
}
|
||||
case "dns", "tls":
|
||||
if m.Target == "" || strings.ContainsAny(m.Target, " /\\") {
|
||||
return errors.New("enter a hostname")
|
||||
}
|
||||
case "heartbeat":
|
||||
if m.IntervalS < 10 {
|
||||
return errors.New("a heartbeat interval is at least 10 seconds")
|
||||
}
|
||||
if m.Token == "" {
|
||||
m.Token = newToken()
|
||||
}
|
||||
default:
|
||||
return errors.New("kind must be http, tcp, dns, tls or heartbeat")
|
||||
}
|
||||
if m.Severity == "" {
|
||||
m.Severity = "critical"
|
||||
}
|
||||
if _, _, err := parseRange(m.Expect); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseRange reads "200-299", "204" or "" (= 200-299).
|
||||
func parseRange(s string) (lo, hi int, err error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 200, 299, nil
|
||||
}
|
||||
a, b, found := strings.Cut(s, "-")
|
||||
if lo, err = strconv.Atoi(strings.TrimSpace(a)); err != nil {
|
||||
return 0, 0, errors.New(`expected status must look like "200-299" or "204"`)
|
||||
}
|
||||
hi = lo
|
||||
if found {
|
||||
if hi, err = strconv.Atoi(strings.TrimSpace(b)); err != nil || hi < lo {
|
||||
return 0, 0, errors.New(`expected status must look like "200-299" or "204"`)
|
||||
}
|
||||
}
|
||||
return lo, hi, nil
|
||||
}
|
||||
|
||||
// ---------- the checks ----------
|
||||
|
||||
// runCheck performs one probe and returns how long it took and an error if it failed.
|
||||
func (s *Server) runCheck(ctx context.Context, m *Monitor) (time.Duration, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, m.timeout())
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
var err error
|
||||
switch m.Kind {
|
||||
case "http":
|
||||
err = s.checkHTTP(ctx, m)
|
||||
case "tcp":
|
||||
var c net.Conn
|
||||
if c, err = netguard.DialContext(m.AllowPrivate)(ctx, "tcp", m.Target); err == nil {
|
||||
c.Close()
|
||||
}
|
||||
case "dns":
|
||||
var addrs []string
|
||||
if addrs, err = net.DefaultResolver.LookupHost(ctx, m.Target); err == nil && len(addrs) == 0 {
|
||||
err = errors.New("no addresses")
|
||||
}
|
||||
case "tls":
|
||||
err = checkTLS(ctx, m, s.now())
|
||||
}
|
||||
return time.Since(start), err
|
||||
}
|
||||
|
||||
func (s *Server) checkHTTP(ctx context.Context, m *Monitor) error {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", m.Target, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Sandbar-Monitor/1")
|
||||
c := *s.httpClient(m.AllowPrivate)
|
||||
c.Timeout = 0 // the context carries the deadline
|
||||
c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
lo, hi, _ := parseRange(m.Expect)
|
||||
if resp.StatusCode < lo || resp.StatusCode > hi {
|
||||
return fmt.Errorf("status %d, expected %d-%d", resp.StatusCode, lo, hi)
|
||||
}
|
||||
if m.Keyword != "" {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if !strings.Contains(string(body), m.Keyword) {
|
||||
return fmt.Errorf("the page doesn't contain %q", m.Keyword)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkTLS(ctx context.Context, m *Monitor, now time.Time) error {
|
||||
host, port, err := net.SplitHostPort(m.Target)
|
||||
if err != nil {
|
||||
host, port = m.Target, "443"
|
||||
}
|
||||
d := tls.Dialer{Config: &tls.Config{ServerName: host}}
|
||||
conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, port))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
certs := conn.(*tls.Conn).ConnectionState().PeerCertificates
|
||||
if len(certs) == 0 {
|
||||
return errors.New("no certificate")
|
||||
}
|
||||
warn := m.WarnDays
|
||||
if warn <= 0 {
|
||||
warn = 14
|
||||
}
|
||||
left := certs[0].NotAfter.Sub(now)
|
||||
if left < time.Duration(warn)*24*time.Hour {
|
||||
return fmt.Errorf("the certificate expires in %d days (%s)", int(left.Hours()/24), certs[0].NotAfter.Format("2006-01-02"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------- scheduling and state ----------
|
||||
|
||||
func (s *Server) monitorState(id string) MonitorState {
|
||||
st := MonitorState{MonitorID: id, Status: "pending", Uptime30: -1}
|
||||
s.st.db.QueryRow(`SELECT status,since,last_check,last_ms,fails,last_ping,detail FROM mstate WHERE monitor_id=?`, id).
|
||||
Scan(&st.Status, &st.Since, &st.LastCheck, &st.LastMS, &st.Fails, &st.LastPing, &st.Detail)
|
||||
var total, ok sql.NullInt64
|
||||
s.st.db.QueryRow(`SELECT COUNT(*), SUM(ok) FROM checks WHERE monitor_id=?`, id).Scan(&total, &ok)
|
||||
if total.Int64 > 0 {
|
||||
st.Uptime30 = float64(ok.Int64) * 100 / float64(total.Int64)
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
func (s *Server) saveState(st MonitorState) {
|
||||
s.st.db.Exec(`INSERT INTO mstate(monitor_id,status,since,last_check,last_ms,fails,last_ping,detail) VALUES(?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(monitor_id) DO UPDATE SET status=excluded.status, since=excluded.since, last_check=excluded.last_check,
|
||||
last_ms=excluded.last_ms, fails=excluded.fails, last_ping=excluded.last_ping, detail=excluded.detail`,
|
||||
st.MonitorID, st.Status, st.Since, st.LastCheck, st.LastMS, st.Fails, st.LastPing, st.Detail)
|
||||
}
|
||||
|
||||
// runDueMonitors starts a check for every enabled monitor whose interval has elapsed.
|
||||
func (s *Server) runDueMonitors(now time.Time) {
|
||||
mons, err := listOf[Monitor](s.st, "monitor")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for i := range mons {
|
||||
m := mons[i]
|
||||
if !m.Enabled {
|
||||
continue
|
||||
}
|
||||
st := s.monitorState(m.ID)
|
||||
if now.Sub(time.Unix(st.LastCheck, 0)) < m.interval() {
|
||||
continue
|
||||
}
|
||||
s.mu.Lock()
|
||||
busy := s.running[m.ID]
|
||||
s.running[m.ID] = true
|
||||
s.mu.Unlock()
|
||||
if busy {
|
||||
continue
|
||||
}
|
||||
go func() {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
delete(s.running, m.ID)
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
s.checkMonitor(&m, now)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// checkMonitor runs one probe (or evaluates a heartbeat) and moves state forward.
|
||||
func (s *Server) checkMonitor(m *Monitor, now time.Time) {
|
||||
st := s.monitorState(m.ID)
|
||||
var dur time.Duration
|
||||
var err error
|
||||
if m.Kind == "heartbeat" {
|
||||
if st.LastPing == 0 {
|
||||
// Never pinged: give it a full interval from when we first looked.
|
||||
if st.LastCheck == 0 {
|
||||
st.LastCheck, st.Status = now.Unix(), "pending"
|
||||
st.Since = now.Unix()
|
||||
s.saveState(st)
|
||||
return
|
||||
}
|
||||
if now.Unix()-st.Since < int64(m.IntervalS+m.GraceS) {
|
||||
st.LastCheck = now.Unix() // still within the first interval: nothing to judge yet
|
||||
s.saveState(st)
|
||||
return
|
||||
}
|
||||
err = errors.New("no heartbeat received yet")
|
||||
} else if now.Unix()-st.LastPing > int64(m.IntervalS+m.GraceS) {
|
||||
err = fmt.Errorf("no heartbeat for %s", now.Sub(time.Unix(st.LastPing, 0)).Round(time.Second))
|
||||
}
|
||||
} else {
|
||||
dur, err = s.runCheck(context.Background(), m)
|
||||
}
|
||||
s.recordResult(m, st, now, dur, err)
|
||||
}
|
||||
|
||||
func (s *Server) recordResult(m *Monitor, st MonitorState, now time.Time, dur time.Duration, err error) {
|
||||
ok := err == nil
|
||||
okInt := 0
|
||||
if ok {
|
||||
okInt = 1
|
||||
}
|
||||
s.st.db.Exec(`INSERT INTO checks(monitor_id,time,ok,ms,detail) VALUES(?,?,?,?,?)`, m.ID, now.Unix(), okInt, dur.Milliseconds(), errText(err))
|
||||
st.LastCheck, st.LastMS, st.Detail = now.Unix(), int(dur.Milliseconds()), errText(err)
|
||||
key := "monitor:" + m.ID
|
||||
if ok {
|
||||
wasDown := st.Status == "down"
|
||||
if st.Status != "up" {
|
||||
st.Status, st.Since = "up", now.Unix()
|
||||
}
|
||||
st.Fails = 0
|
||||
s.saveState(st)
|
||||
if wasDown {
|
||||
s.ResolveAlertKey(key, "monitor")
|
||||
}
|
||||
return
|
||||
}
|
||||
st.Fails++
|
||||
if st.Fails >= m.retries() && st.Status != "down" {
|
||||
st.Status, st.Since = "down", now.Unix()
|
||||
}
|
||||
s.saveState(st)
|
||||
if st.Status == "down" {
|
||||
s.RaiseAlert(key, "monitor", m.Name+" is down", m.Name+" ("+m.Kind+" "+m.Target+"): "+st.Detail, m.Severity, m.PolicyID)
|
||||
}
|
||||
}
|
||||
|
||||
func errText(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
// Heartbeat records a ping for the monitor with this token. It returns false if there is none.
|
||||
func (s *Server) Heartbeat(token string) bool {
|
||||
if token == "" {
|
||||
return false
|
||||
}
|
||||
mons, _ := listOf[Monitor](s.st, "monitor")
|
||||
for i := range mons {
|
||||
m := mons[i]
|
||||
if m.Kind == "heartbeat" && m.Token == token && m.Enabled {
|
||||
st := s.monitorState(m.ID)
|
||||
st.LastPing = s.now().Unix()
|
||||
if st.Since == 0 {
|
||||
st.Since = st.LastPing
|
||||
}
|
||||
s.saveState(st)
|
||||
// A ping on a down monitor recovers it at once rather than at the next check.
|
||||
if st.Status == "down" {
|
||||
s.recordResult(&m, st, s.now(), 0, nil)
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// deleteMonitorData drops what we keep about a monitor and closes its alert.
|
||||
func (s *Server) deleteMonitorData(id string) {
|
||||
s.st.db.Exec(`DELETE FROM mstate WHERE monitor_id=?`, id)
|
||||
s.st.db.Exec(`DELETE FROM checks WHERE monitor_id=?`, id)
|
||||
s.ResolveAlertKey("monitor:"+id, "monitor deleted")
|
||||
}
|
||||
|
||||
func (s *Server) recentChecks(id string, limit int) []map[string]any {
|
||||
out := []map[string]any{}
|
||||
rows, err := s.st.db.Query(`SELECT time,ok,ms,detail FROM checks WHERE monitor_id=? ORDER BY time DESC LIMIT ?`, id, limit)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var t, ok, ms int64
|
||||
var d string
|
||||
if rows.Scan(&t, &ok, &ms, &d) == nil {
|
||||
out = append(out, map[string]any{"time": t, "ok": ok == 1, "ms": ms, "detail": d})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/smtp"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.veillax.com/Veillax/reef/netguard"
|
||||
)
|
||||
|
||||
// Notify is the ntfy-shaped part: anything can publish a message to a topic with
|
||||
// an HTTP request, and anything can subscribe to it (SSE, newline-delimited JSON,
|
||||
// or the web UI). Alerts and incidents publish here too, so a phone with the ntfy
|
||||
// app subscribed to "alerts" is a working pager with no other setup.
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
Topic string `json:"topic"`
|
||||
Time int64 `json:"time"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Message string `json:"message"`
|
||||
Priority int `json:"priority"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
var topicRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
|
||||
|
||||
// reservedTopics are first path segments that belong to Sandbar itself.
|
||||
var reservedTopics = map[string]bool{"api": true, "status": true, "hb": true, "metrics": true, "static": true, "healthz": true}
|
||||
|
||||
func validTopic(t string) bool { return topicRe.MatchString(t) && !reservedTopics[t] }
|
||||
|
||||
// hub fans published messages out to live subscribers.
|
||||
type hub struct {
|
||||
mu sync.Mutex
|
||||
subs map[string]map[chan Message]bool
|
||||
}
|
||||
|
||||
func newHub() *hub { return &hub{subs: map[string]map[chan Message]bool{}} }
|
||||
|
||||
func (h *hub) subscribe(topic string) (chan Message, func()) {
|
||||
ch := make(chan Message, 64)
|
||||
h.mu.Lock()
|
||||
if h.subs[topic] == nil {
|
||||
h.subs[topic] = map[chan Message]bool{}
|
||||
}
|
||||
h.subs[topic][ch] = true
|
||||
h.mu.Unlock()
|
||||
return ch, func() {
|
||||
h.mu.Lock()
|
||||
delete(h.subs[topic], ch)
|
||||
h.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (h *hub) publish(m Message) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for ch := range h.subs[m.Topic] {
|
||||
select {
|
||||
case ch <- m:
|
||||
default: // a stalled subscriber drops messages instead of blocking publishers
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parsePriority(s string) int {
|
||||
switch strings.ToLower(strings.TrimSpace(s)) {
|
||||
case "5", "max", "urgent":
|
||||
return 5
|
||||
case "4", "high":
|
||||
return 4
|
||||
case "2", "low":
|
||||
return 2
|
||||
case "1", "min":
|
||||
return 1
|
||||
}
|
||||
return 3
|
||||
}
|
||||
|
||||
func parseTags(s string) []string {
|
||||
var out []string
|
||||
for _, t := range strings.Split(s, ",") {
|
||||
if t = strings.TrimSpace(t); t != "" {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Publish stores a message and delivers it to live subscribers.
|
||||
func (s *Server) Publish(topic, title, body string, prio int, tags []string) (Message, error) {
|
||||
if !validTopic(topic) {
|
||||
return Message{}, errors.New("a topic is 1-64 letters, digits, - or _")
|
||||
}
|
||||
m := Message{ID: newID(), Topic: topic, Time: s.now().Unix(), Title: title, Message: body, Priority: prio, Tags: tags}
|
||||
tj, _ := json.Marshal(tags)
|
||||
if _, err := s.st.db.Exec(`INSERT INTO messages(id,topic,time,title,body,priority,tags) VALUES(?,?,?,?,?,?,?)`,
|
||||
m.ID, m.Topic, m.Time, m.Title, m.Message, m.Priority, string(tj)); err != nil {
|
||||
return m, err
|
||||
}
|
||||
s.hub.publish(m)
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// MessagesSince returns stored messages for a topic after the given unix time, oldest first.
|
||||
func (s *Server) MessagesSince(topic string, since int64, limit int) []Message {
|
||||
out := []Message{}
|
||||
rows, err := s.st.db.Query(`SELECT id,time,title,body,priority,tags FROM messages WHERE topic=? AND time>=? ORDER BY time, rowid LIMIT ?`, topic, since, limit)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
m := Message{Topic: topic}
|
||||
var tj string
|
||||
if rows.Scan(&m.ID, &m.Time, &m.Title, &m.Message, &m.Priority, &tj) == nil {
|
||||
json.Unmarshal([]byte(tj), &m.Tags)
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------- outputs: email and webhooks ----------
|
||||
|
||||
// Channel is a place broadcast notifications go: an email address list or a webhook.
|
||||
type Channel struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"` // "email" or "webhook"
|
||||
Target string `json:"target"` // addresses, or a URL (secret: URLs often carry tokens)
|
||||
Format string `json:"format"` // webhook body: json, slack, discord or ntfy
|
||||
AllowPrivate bool `json:"allowPrivate"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// Note is what every notification is built from.
|
||||
type Note struct {
|
||||
Status string `json:"status"` // firing, resolved, acked, test, declared, update
|
||||
Kind string `json:"kind"` // alert, monitor, incident
|
||||
Title string `json:"title"`
|
||||
Message string `json:"message"`
|
||||
Severity string `json:"severity,omitempty"`
|
||||
Link string `json:"link,omitempty"`
|
||||
Time int64 `json:"time"`
|
||||
}
|
||||
|
||||
func (n Note) subject() string {
|
||||
switch n.Status {
|
||||
case "resolved":
|
||||
return "[Sandbar] Resolved: " + n.Title
|
||||
case "test":
|
||||
return "[Sandbar] Test notification"
|
||||
case "acked":
|
||||
return "[Sandbar] Acknowledged: " + n.Title
|
||||
}
|
||||
return "[Sandbar] " + strings.ToUpper(n.Kind[:1]) + n.Kind[1:] + ": " + n.Title
|
||||
}
|
||||
|
||||
func (n Note) text() string {
|
||||
if n.Status == "test" {
|
||||
return "This is a test from Sandbar. If you can read this, the channel works."
|
||||
}
|
||||
t := n.Message
|
||||
if n.Link != "" {
|
||||
t += "\n" + n.Link
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func (n Note) priority() int {
|
||||
switch {
|
||||
case n.Status == "resolved" || n.Status == "acked" || n.Status == "test":
|
||||
return 3
|
||||
case n.Severity == "critical":
|
||||
return 5
|
||||
}
|
||||
return 4
|
||||
}
|
||||
|
||||
func (n Note) tags() []string {
|
||||
switch n.Status {
|
||||
case "resolved":
|
||||
return []string{"white_check_mark"}
|
||||
case "acked":
|
||||
return []string{"eyes"}
|
||||
case "test":
|
||||
return []string{"test_tube"}
|
||||
}
|
||||
return []string{"rotating_light"}
|
||||
}
|
||||
|
||||
func webhookBody(format string, n Note) (body []byte, ctype string, hdr map[string]string) {
|
||||
text := n.subject() + "\n" + n.text()
|
||||
switch format {
|
||||
case "slack":
|
||||
body, _ = json.Marshal(map[string]string{"text": text})
|
||||
case "discord":
|
||||
body, _ = json.Marshal(map[string]string{"content": text})
|
||||
case "ntfy":
|
||||
return []byte(n.text()), "text/plain", map[string]string{
|
||||
"Title": n.subject(), "Priority": strconv.Itoa(n.priority()), "Tags": strings.Join(n.tags(), ","),
|
||||
}
|
||||
default:
|
||||
body, _ = json.Marshal(n)
|
||||
}
|
||||
return body, "application/json", nil
|
||||
}
|
||||
|
||||
func (s *Server) sendChannel(ch Channel, n Note) error {
|
||||
switch ch.Kind {
|
||||
case "email":
|
||||
var errs []error
|
||||
for _, to := range strings.Split(s.st.open(ch.Target), ",") {
|
||||
if to = strings.TrimSpace(to); to != "" {
|
||||
if err := s.Mail(to, n.subject(), n.text()); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
case "webhook":
|
||||
body, ctype, hdr := webhookBody(ch.Format, n)
|
||||
req, err := http.NewRequest("POST", s.st.open(ch.Target), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", ctype)
|
||||
for k, v := range hdr {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := s.httpClient(ch.AllowPrivate).Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("the webhook answered %s", resp.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("unknown channel kind")
|
||||
}
|
||||
|
||||
func (s *Server) httpClient(allowPrivate bool) *http.Client {
|
||||
if s.opts.HTTPClient != nil {
|
||||
return s.opts.HTTPClient
|
||||
}
|
||||
c := netguard.Client(allowPrivate)
|
||||
c.Timeout = 15 * time.Second
|
||||
return c
|
||||
}
|
||||
|
||||
// Mail sends one plain-text email through the configured SMTP server.
|
||||
func (s *Server) Mail(to, subject, body string) error {
|
||||
if s.opts.Mailer != nil {
|
||||
return s.opts.Mailer(to, subject, body)
|
||||
}
|
||||
c := s.opts.SMTP
|
||||
if c.Host == "" {
|
||||
return errors.New("no SMTP server configured (SANDBAR_SMTP_HOST)")
|
||||
}
|
||||
if strings.ContainsAny(to+subject, "\r\n") {
|
||||
return errors.New("invalid header")
|
||||
}
|
||||
port := c.Port
|
||||
if port == "" {
|
||||
port = "587"
|
||||
}
|
||||
msg := "From: " + c.From + "\r\nTo: " + to + "\r\nSubject: " + subject +
|
||||
"\r\nDate: " + s.now().Format(time.RFC1123Z) + "\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n" + body + "\r\n"
|
||||
var auth smtp.Auth
|
||||
if c.User != "" {
|
||||
auth = smtp.PlainAuth("", c.User, c.Password, c.Host)
|
||||
}
|
||||
return smtp.SendMail(c.Host+":"+port, auth, c.From, []string{to}, []byte(msg))
|
||||
}
|
||||
|
||||
// broadcast sends a note to the topic named by the source and every enabled channel.
|
||||
func (s *Server) broadcast(n Note, topics ...string) {
|
||||
for _, t := range topics {
|
||||
if _, err := s.Publish(t, n.subject(), n.text(), n.priority(), n.tags()); err != nil {
|
||||
log.Printf("publish %s: %v", t, err)
|
||||
}
|
||||
}
|
||||
chans, _ := listOf[Channel](s.st, "channel")
|
||||
for _, ch := range chans {
|
||||
if !ch.Enabled {
|
||||
continue
|
||||
}
|
||||
ch := ch
|
||||
go func() {
|
||||
if err := s.sendChannel(ch, n); err != nil {
|
||||
log.Printf("channel %q: %v", ch.Name, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// validateChannel normalises ch.
|
||||
func validateChannel(ch *Channel) error {
|
||||
ch.Name = strings.TrimSpace(ch.Name)
|
||||
ch.Target = strings.TrimSpace(ch.Target)
|
||||
switch ch.Kind {
|
||||
case "email":
|
||||
for _, a := range strings.Split(ch.Target, ",") {
|
||||
if a = strings.TrimSpace(a); !strings.Contains(a, "@") || strings.ContainsAny(a, " \r\n<>") {
|
||||
return errors.New("enter one or more email addresses, separated by commas")
|
||||
}
|
||||
}
|
||||
ch.Format, ch.AllowPrivate = "", false
|
||||
case "webhook":
|
||||
u, err := url.Parse(ch.Target)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return errors.New("the webhook must be an http:// or https:// URL")
|
||||
}
|
||||
switch ch.Format {
|
||||
case "json", "slack", "discord", "ntfy":
|
||||
case "":
|
||||
ch.Format = "json"
|
||||
default:
|
||||
return errors.New("format must be json, slack, discord or ntfy")
|
||||
}
|
||||
default:
|
||||
return errors.New("a channel is an email or a webhook")
|
||||
}
|
||||
if ch.Name == "" {
|
||||
ch.Name = ch.Kind
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// On-call: who is responsible right now, and what happens to an alert nobody has
|
||||
// picked up. An alert follows its escalation policy, a list of steps, each of
|
||||
// which waits a delay and then notifies people and/or whoever is on call for a
|
||||
// schedule. Acknowledging or resolving the alert stops the walk. Alerts are kept
|
||||
// in the database with the time the next step is due, so a restart neither loses
|
||||
// nor repeats them.
|
||||
|
||||
type Override struct {
|
||||
UserID string `json:"userId"`
|
||||
From int64 `json:"from"`
|
||||
To int64 `json:"to"`
|
||||
}
|
||||
|
||||
// Schedule rotates through Members, handing over every RotationHours from Start.
|
||||
type Schedule struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Members []string `json:"members"` // user ids, in rotation order
|
||||
RotationHours int `json:"rotationHours"`
|
||||
Start int64 `json:"start"`
|
||||
Overrides []Override `json:"overrides"`
|
||||
}
|
||||
|
||||
// OnCall returns the user id on call at t, or "".
|
||||
func (sc *Schedule) OnCall(t time.Time) string {
|
||||
for i := len(sc.Overrides) - 1; i >= 0; i-- {
|
||||
o := sc.Overrides[i]
|
||||
if t.Unix() >= o.From && t.Unix() < o.To {
|
||||
return o.UserID
|
||||
}
|
||||
}
|
||||
if len(sc.Members) == 0 {
|
||||
return ""
|
||||
}
|
||||
rot := int64(sc.RotationHours) * 3600
|
||||
if rot <= 0 {
|
||||
return sc.Members[0]
|
||||
}
|
||||
elapsed := t.Unix() - sc.Start
|
||||
if elapsed < 0 {
|
||||
return sc.Members[0]
|
||||
}
|
||||
return sc.Members[(elapsed/rot)%int64(len(sc.Members))]
|
||||
}
|
||||
|
||||
type Step struct {
|
||||
DelayMin int `json:"delayMin"` // wait this long after the previous step (0 = at once)
|
||||
Users []string `json:"users"`
|
||||
Schedules []string `json:"schedules"`
|
||||
}
|
||||
|
||||
type Policy struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Steps []Step `json:"steps"`
|
||||
RepeatMin int `json:"repeatMin"` // after the last step, start over this many minutes later (0 = never)
|
||||
}
|
||||
|
||||
type Alert struct {
|
||||
ID string `json:"id"`
|
||||
Key string `json:"key"`
|
||||
Source string `json:"source"`
|
||||
Summary string `json:"summary"`
|
||||
Detail string `json:"detail"`
|
||||
Severity string `json:"severity"` // critical, warning, info
|
||||
Status string `json:"status"` // firing, acked, resolved
|
||||
PolicyID string `json:"policyId"`
|
||||
Step int `json:"step"`
|
||||
NextAt int64 `json:"nextAt"`
|
||||
Created int64 `json:"created"`
|
||||
Updated int64 `json:"updated"`
|
||||
Count int `json:"count"`
|
||||
AckedBy string `json:"ackedBy"`
|
||||
AckedAt int64 `json:"ackedAt"`
|
||||
SnoozeUntil int64 `json:"snoozeUntil"`
|
||||
ResolvedAt int64 `json:"resolvedAt"`
|
||||
IncidentID string `json:"incidentId"`
|
||||
Timeline []Event `json:"timeline,omitempty"`
|
||||
}
|
||||
|
||||
const alertCols = `id,key,source,summary,detail,severity,status,policy_id,step,next_at,created,updated,count,acked_by,acked_at,snooze_until,resolved_at,incident_id`
|
||||
|
||||
type scanner interface{ Scan(...any) error }
|
||||
|
||||
func scanAlert(r scanner) (*Alert, error) {
|
||||
var a Alert
|
||||
err := r.Scan(&a.ID, &a.Key, &a.Source, &a.Summary, &a.Detail, &a.Severity, &a.Status, &a.PolicyID, &a.Step, &a.NextAt,
|
||||
&a.Created, &a.Updated, &a.Count, &a.AckedBy, &a.AckedAt, &a.SnoozeUntil, &a.ResolvedAt, &a.IncidentID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, errNotFound
|
||||
}
|
||||
return &a, err
|
||||
}
|
||||
|
||||
func (s *Server) getAlert(id string) (*Alert, error) {
|
||||
return scanAlert(s.st.db.QueryRow(`SELECT `+alertCols+` FROM alerts WHERE id=?`, id))
|
||||
}
|
||||
|
||||
func (s *Server) listAlerts(status string, limit int) []Alert {
|
||||
q := `SELECT ` + alertCols + ` FROM alerts`
|
||||
var args []any
|
||||
switch status {
|
||||
case "open":
|
||||
q += ` WHERE status IN ('firing','acked')`
|
||||
case "firing", "acked", "resolved":
|
||||
q += ` WHERE status=?`
|
||||
args = append(args, status)
|
||||
}
|
||||
q += ` ORDER BY created DESC LIMIT ?`
|
||||
args = append(args, limit)
|
||||
out := []Alert{}
|
||||
rows, err := s.st.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
if a, err := scanAlert(rows); err == nil {
|
||||
out = append(out, *a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RaiseAlert opens an alert for key, or bumps the open one with the same key
|
||||
// (deduplication). It returns the alert and whether it is new.
|
||||
func (s *Server) RaiseAlert(key, source, summary, detail, severity, policyID string) (*Alert, bool, error) {
|
||||
now := s.now()
|
||||
if severity == "" {
|
||||
severity = "warning"
|
||||
}
|
||||
existing, err := scanAlert(s.st.db.QueryRow(`SELECT `+alertCols+` FROM alerts WHERE key=? AND status IN ('firing','acked') ORDER BY created DESC LIMIT 1`, key))
|
||||
if err == nil {
|
||||
existing.Count++
|
||||
existing.Updated = now.Unix()
|
||||
existing.Summary, existing.Detail = summary, detail
|
||||
_, err := s.st.db.Exec(`UPDATE alerts SET count=?, updated=?, summary=?, detail=? WHERE id=?`, existing.Count, existing.Updated, summary, detail, existing.ID)
|
||||
return existing, false, err
|
||||
}
|
||||
if !errors.Is(err, errNotFound) {
|
||||
return nil, false, err
|
||||
}
|
||||
a := &Alert{ID: newID(), Key: key, Source: source, Summary: summary, Detail: detail, Severity: severity, Status: "firing",
|
||||
PolicyID: policyID, NextAt: now.Unix(), Created: now.Unix(), Updated: now.Unix(), Count: 1}
|
||||
if policyID == "" {
|
||||
a.NextAt = 0 // nothing to escalate: it is only broadcast
|
||||
}
|
||||
_, err = s.st.db.Exec(`INSERT INTO alerts(`+alertCols+`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
a.ID, a.Key, a.Source, a.Summary, a.Detail, a.Severity, a.Status, a.PolicyID, a.Step, a.NextAt, a.Created, a.Updated, a.Count,
|
||||
a.AckedBy, a.AckedAt, a.SnoozeUntil, a.ResolvedAt, a.IncidentID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
s.st.addEvent("alert", a.ID, "created", source, summary, now)
|
||||
s.broadcast(Note{Status: "firing", Kind: "alert", Title: summary, Message: s.alertText(a), Severity: severity, Link: s.link("/#alerts"), Time: now.Unix()}, "alerts")
|
||||
return a, true, nil
|
||||
}
|
||||
|
||||
func (s *Server) alertText(a *Alert) string {
|
||||
t := strings.ToUpper(a.Severity) + ": " + a.Summary
|
||||
if a.Detail != "" {
|
||||
t += "\n" + a.Detail
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// ResolveAlertKey resolves the open alert for key, if any.
|
||||
func (s *Server) ResolveAlertKey(key, actor string) {
|
||||
var id string
|
||||
if s.st.db.QueryRow(`SELECT id FROM alerts WHERE key=? AND status IN ('firing','acked') LIMIT 1`, key).Scan(&id) == nil {
|
||||
s.ResolveAlert(id, actor)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) ResolveAlert(id, actor string) error {
|
||||
a, err := s.getAlert(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Status == "resolved" {
|
||||
return nil
|
||||
}
|
||||
now := s.now()
|
||||
if _, err := s.st.db.Exec(`UPDATE alerts SET status='resolved', next_at=0, resolved_at=?, updated=? WHERE id=?`, now.Unix(), now.Unix(), id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.st.addEvent("alert", id, "resolved", actor, "resolved", now)
|
||||
s.broadcast(Note{Status: "resolved", Kind: "alert", Title: a.Summary, Message: "Resolved: " + a.Summary, Severity: a.Severity, Link: s.link("/#alerts"), Time: now.Unix()}, "alerts")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) AckAlert(id, actor string) error {
|
||||
a, err := s.getAlert(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Status != "firing" {
|
||||
return errors.New("only a firing alert can be acknowledged")
|
||||
}
|
||||
now := s.now()
|
||||
if _, err := s.st.db.Exec(`UPDATE alerts SET status='acked', next_at=0, acked_by=?, acked_at=?, updated=? WHERE id=?`, actor, now.Unix(), now.Unix(), id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.st.addEvent("alert", id, "acked", actor, "acknowledged", now)
|
||||
s.broadcast(Note{Status: "acked", Kind: "alert", Title: a.Summary, Message: "Acknowledged by " + actor + ": " + a.Summary, Time: now.Unix()}, "alerts")
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnoozeAlert pauses escalation for d. When it ends the walk resumes at the step it was on.
|
||||
func (s *Server) SnoozeAlert(id, actor string, d time.Duration) error {
|
||||
a, err := s.getAlert(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Status != "firing" {
|
||||
return errors.New("only a firing alert can be snoozed")
|
||||
}
|
||||
until := s.now().Add(d).Unix()
|
||||
if _, err := s.st.db.Exec(`UPDATE alerts SET next_at=?, snooze_until=? WHERE id=?`, until, until, id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.st.addEvent("alert", id, "snoozed", actor, "snoozed for "+d.String(), s.now())
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------- escalation ----------
|
||||
|
||||
// escalate runs every alert step that has come due.
|
||||
func (s *Server) escalate(now time.Time) {
|
||||
rows, err := s.st.db.Query(`SELECT id FROM alerts WHERE status='firing' AND next_at>0 AND next_at<=?`, now.Unix())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if rows.Scan(&id) == nil {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
for _, id := range ids {
|
||||
s.runStep(id, now)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) runStep(id string, now time.Time) {
|
||||
a, err := s.getAlert(id)
|
||||
if err != nil || a.Status != "firing" {
|
||||
return
|
||||
}
|
||||
pol, err := getOf[Policy](s.st, "policy", a.PolicyID)
|
||||
if err != nil || len(pol.Steps) == 0 {
|
||||
s.st.db.Exec(`UPDATE alerts SET next_at=0 WHERE id=?`, id) // policy deleted: stop trying
|
||||
s.st.addEvent("alert", id, "note", "sandbar", "its escalation policy is gone or empty", now)
|
||||
return
|
||||
}
|
||||
if a.Step >= len(pol.Steps) {
|
||||
a.Step = 0
|
||||
}
|
||||
step := pol.Steps[a.Step]
|
||||
var people []string
|
||||
seen := map[string]bool{}
|
||||
add := func(uid string) {
|
||||
if uid != "" && !seen[uid] {
|
||||
seen[uid] = true
|
||||
people = append(people, uid)
|
||||
}
|
||||
}
|
||||
for _, u := range step.Users {
|
||||
add(u)
|
||||
}
|
||||
for _, scID := range step.Schedules {
|
||||
if sc, err := getOf[Schedule](s.st, "schedule", scID); err == nil {
|
||||
add(sc.OnCall(now))
|
||||
}
|
||||
}
|
||||
var names []string
|
||||
for _, uid := range people {
|
||||
if u, err := s.userByID(uid); err == nil {
|
||||
s.notifyUser(u, Note{Status: "firing", Kind: "alert", Title: a.Summary, Message: s.alertText(a), Severity: a.Severity,
|
||||
Link: s.link("/#alerts"), Time: now.Unix()})
|
||||
names = append(names, u.Name)
|
||||
}
|
||||
}
|
||||
text := fmt.Sprintf("step %d: notified %s", a.Step+1, strings.Join(names, ", "))
|
||||
if len(names) == 0 {
|
||||
text = fmt.Sprintf("step %d: nobody to notify (empty schedule or removed users)", a.Step+1)
|
||||
}
|
||||
s.st.addEvent("alert", id, "escalated", "sandbar", text, now)
|
||||
|
||||
next := a.Step + 1
|
||||
var nextAt int64
|
||||
switch {
|
||||
case next < len(pol.Steps):
|
||||
nextAt = now.Add(time.Duration(pol.Steps[next].DelayMin) * time.Minute).Unix()
|
||||
case pol.RepeatMin > 0:
|
||||
next = 0
|
||||
nextAt = now.Add(time.Duration(pol.RepeatMin) * time.Minute).Unix()
|
||||
}
|
||||
s.st.db.Exec(`UPDATE alerts SET step=?, next_at=?, updated=? WHERE id=?`, next, nextAt, now.Unix(), id)
|
||||
}
|
||||
|
||||
// notifyUser reaches one person: a message on their personal topic (u-<id>, which
|
||||
// their phone subscribes to) and an email if the server can send mail.
|
||||
func (s *Server) notifyUser(u *User, n Note) {
|
||||
s.Publish(userTopic(u.ID), n.subject(), n.text(), n.priority(), n.tags())
|
||||
if u.Email != "" && (s.opts.SMTP.Host != "" || s.opts.Mailer != nil) {
|
||||
go func() {
|
||||
if err := s.Mail(u.Email, n.subject(), n.text()); err != nil {
|
||||
s.st.addEvent("alert", "", "error", "sandbar", "email to "+u.Email+": "+err.Error(), s.now())
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func userTopic(id string) string { return "u-" + id }
|
||||
@@ -0,0 +1,550 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.veillax.com/Veillax/reef/secretbox"
|
||||
)
|
||||
|
||||
type fakeClock struct {
|
||||
mu sync.Mutex
|
||||
t time.Time
|
||||
}
|
||||
|
||||
func (c *fakeClock) Now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t }
|
||||
func (c *fakeClock) Advance(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
c.t = c.t.Add(d)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
type mailLog struct {
|
||||
mu sync.Mutex
|
||||
sent []string
|
||||
}
|
||||
|
||||
func (m *mailLog) send(to, subject, body string) error {
|
||||
m.mu.Lock()
|
||||
m.sent = append(m.sent, to)
|
||||
m.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mailLog) to() []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return append([]string(nil), m.sent...)
|
||||
}
|
||||
|
||||
func newTestServer(t *testing.T) (*Server, *fakeClock, *mailLog) {
|
||||
t.Helper()
|
||||
box, _ := secretbox.New(secretbox.GenerateKey())
|
||||
st, err := OpenStore(t.TempDir(), box)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
clk := &fakeClock{t: time.Date(2026, 1, 5, 9, 0, 0, 0, time.UTC)}
|
||||
ml := &mailLog{}
|
||||
return New(Options{Now: clk.Now, Mailer: ml.send, URL: "http://sandbar.test"}, st), clk, ml
|
||||
}
|
||||
|
||||
func mustUser(t *testing.T, s *Server, name string) *User {
|
||||
t.Helper()
|
||||
u, err := s.createUser(name, strings.ToLower(name)+"@example.com", "correct-horse-battery", "member", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
func TestScheduleRotationAndOverride(t *testing.T) {
|
||||
start := time.Date(2026, 1, 5, 0, 0, 0, 0, time.UTC)
|
||||
sc := &Schedule{Members: []string{"a", "b", "c"}, RotationHours: 24, Start: start.Unix()}
|
||||
for day, want := range []string{"a", "b", "c", "a"} {
|
||||
if got := sc.OnCall(start.Add(time.Duration(day)*24*time.Hour + time.Hour)); got != want {
|
||||
t.Errorf("day %d: on call %q, want %q", day, got, want)
|
||||
}
|
||||
}
|
||||
sc.Overrides = []Override{{UserID: "z", From: start.Add(2 * time.Hour).Unix(), To: start.Add(5 * time.Hour).Unix()}}
|
||||
if got := sc.OnCall(start.Add(3 * time.Hour)); got != "z" {
|
||||
t.Errorf("override: %q", got)
|
||||
}
|
||||
if got := sc.OnCall(start.Add(6 * time.Hour)); got != "a" {
|
||||
t.Errorf("after override: %q", got)
|
||||
}
|
||||
if (&Schedule{}).OnCall(start) != "" {
|
||||
t.Error("empty schedule should have nobody on call")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscalationWalksStepsUntilAcked(t *testing.T) {
|
||||
s, clk, mails := newTestServer(t)
|
||||
alice, bob := mustUser(t, s, "Alice"), mustUser(t, s, "Bob")
|
||||
s.st.put("schedule", "sch", &Schedule{ID: "sch", Name: "primary", Members: []string{alice.ID}, RotationHours: 24, Start: clk.Now().Unix()})
|
||||
s.st.put("policy", "pol", &Policy{ID: "pol", Name: "p", Steps: []Step{
|
||||
{Schedules: []string{"sch"}},
|
||||
{DelayMin: 5, Users: []string{bob.ID}},
|
||||
}})
|
||||
|
||||
a, isNew, err := s.RaiseAlert("k1", "test", "db down", "", "critical", "pol")
|
||||
if err != nil || !isNew {
|
||||
t.Fatalf("raise: %v new=%v", err, isNew)
|
||||
}
|
||||
s.escalate(clk.Now())
|
||||
if got := s.MessagesSince(userTopic(alice.ID), 0, 10); len(got) != 1 {
|
||||
t.Fatalf("alice should be paged once, got %d", len(got))
|
||||
}
|
||||
if len(s.MessagesSince(userTopic(bob.ID), 0, 10)) != 0 {
|
||||
t.Fatal("bob must wait for step 2")
|
||||
}
|
||||
|
||||
clk.Advance(4 * time.Minute)
|
||||
s.escalate(clk.Now())
|
||||
if len(s.MessagesSince(userTopic(bob.ID), 0, 10)) != 0 {
|
||||
t.Fatal("step 2 fired early")
|
||||
}
|
||||
clk.Advance(2 * time.Minute)
|
||||
s.escalate(clk.Now())
|
||||
if len(s.MessagesSince(userTopic(bob.ID), 0, 10)) != 1 {
|
||||
t.Fatal("bob should be paged after the delay")
|
||||
}
|
||||
|
||||
// A second alert with the same key is deduplicated, not a new page.
|
||||
b, isNew, _ := s.RaiseAlert("k1", "test", "db down", "still", "critical", "pol")
|
||||
if isNew || b.ID != a.ID || b.Count != 2 {
|
||||
t.Fatalf("dedupe: new=%v same=%v count=%d", isNew, b.ID == a.ID, b.Count)
|
||||
}
|
||||
|
||||
// Ack stops escalation entirely.
|
||||
c, _, _ := s.RaiseAlert("k2", "test", "disk full", "", "warning", "pol")
|
||||
s.escalate(clk.Now())
|
||||
if err := s.AckAlert(c.ID, "Alice"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
clk.Advance(time.Hour)
|
||||
before := len(s.MessagesSince(userTopic(bob.ID), 0, 50))
|
||||
s.escalate(clk.Now())
|
||||
if len(s.MessagesSince(userTopic(bob.ID), 0, 50)) != before {
|
||||
t.Fatal("an acked alert kept escalating")
|
||||
}
|
||||
// Emails go out on a goroutine; give them a moment.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for len(mails.to()) == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if len(mails.to()) == 0 {
|
||||
t.Fatal("expected email notifications to be sent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnoozeResumesAtSameStep(t *testing.T) {
|
||||
s, clk, _ := newTestServer(t)
|
||||
alice := mustUser(t, s, "Alice")
|
||||
s.st.put("policy", "pol", &Policy{ID: "pol", Name: "p", Steps: []Step{{Users: []string{alice.ID}}}})
|
||||
a, _, _ := s.RaiseAlert("k", "test", "x", "", "info", "pol")
|
||||
if err := s.SnoozeAlert(a.ID, "me", 30*time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.escalate(clk.Now())
|
||||
if len(s.MessagesSince(userTopic(alice.ID), 0, 10)) != 0 {
|
||||
t.Fatal("snoozed alert paged")
|
||||
}
|
||||
clk.Advance(31 * time.Minute)
|
||||
s.escalate(clk.Now())
|
||||
if len(s.MessagesSince(userTopic(alice.ID), 0, 10)) != 1 {
|
||||
t.Fatal("alert should page when the snooze ends")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorHTTPDownAndRecover(t *testing.T) {
|
||||
s, clk, _ := newTestServer(t)
|
||||
var mu sync.Mutex
|
||||
status := 200
|
||||
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
w.WriteHeader(status)
|
||||
io.WriteString(w, "hello world")
|
||||
}))
|
||||
defer backend.Close()
|
||||
m := &Monitor{ID: "m1", Name: "web", Kind: "http", Target: backend.URL, IntervalS: 30, Retries: 2, Enabled: true, AllowPrivate: true, Keyword: "hello"}
|
||||
if err := validateMonitor(m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.st.put("monitor", m.ID, m)
|
||||
|
||||
step := func() { s.checkMonitor(m, clk.Now()); clk.Advance(30 * time.Second) }
|
||||
setStatus := func(v int) { mu.Lock(); status = v; mu.Unlock() }
|
||||
|
||||
step()
|
||||
if st := s.monitorState("m1"); st.Status != "up" {
|
||||
t.Fatalf("want up, got %s (%s)", st.Status, st.Detail)
|
||||
}
|
||||
setStatus(500)
|
||||
step()
|
||||
if st := s.monitorState("m1"); st.Status != "up" || st.Fails != 1 {
|
||||
t.Fatalf("one failure must not flip it: %+v", st)
|
||||
}
|
||||
step()
|
||||
if st := s.monitorState("m1"); st.Status != "down" {
|
||||
t.Fatalf("want down after retries, got %+v", st)
|
||||
}
|
||||
if open := s.listAlerts("open", 10); len(open) != 1 || !strings.Contains(open[0].Summary, "web is down") {
|
||||
t.Fatalf("expected one open alert, got %+v", open)
|
||||
}
|
||||
step() // still down: same alert, not a second one
|
||||
if open := s.listAlerts("open", 10); len(open) != 1 {
|
||||
t.Fatalf("alert duplicated: %d", len(open))
|
||||
}
|
||||
setStatus(200)
|
||||
step()
|
||||
if st := s.monitorState("m1"); st.Status != "up" {
|
||||
t.Fatalf("want recovered, got %+v", st)
|
||||
}
|
||||
if open := s.listAlerts("open", 10); len(open) != 0 {
|
||||
t.Fatalf("alert should have resolved: %+v", open)
|
||||
}
|
||||
if up := s.monitorState("m1").Uptime30; up <= 0 || up >= 100 {
|
||||
t.Fatalf("uptime should reflect the outage, got %v", up)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorBlocksPrivateTargetsByDefault(t *testing.T) {
|
||||
s, _, _ := newTestServer(t)
|
||||
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer backend.Close()
|
||||
m := &Monitor{Name: "x", Kind: "http", Target: backend.URL}
|
||||
if _, err := s.runCheck(context.Background(), m); err == nil {
|
||||
t.Fatal("a loopback target must be refused unless allowPrivate is set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeatMissed(t *testing.T) {
|
||||
s, clk, _ := newTestServer(t)
|
||||
m := &Monitor{ID: "hb", Name: "backup", Kind: "heartbeat", IntervalS: 60, GraceS: 10, Enabled: true}
|
||||
if err := validateMonitor(m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.st.put("monitor", m.ID, m)
|
||||
s.checkMonitor(m, clk.Now()) // first look: starts the clock
|
||||
if s.monitorState("hb").Status != "pending" {
|
||||
t.Fatal("a heartbeat that was never expected yet should be pending")
|
||||
}
|
||||
clk.Advance(30 * time.Second)
|
||||
if !s.Heartbeat(m.Token) {
|
||||
t.Fatal("ping rejected")
|
||||
}
|
||||
s.checkMonitor(m, clk.Now())
|
||||
if s.monitorState("hb").Status != "up" {
|
||||
t.Fatal("want up after a ping")
|
||||
}
|
||||
clk.Advance(2 * time.Minute)
|
||||
for i := 0; i < 2; i++ {
|
||||
s.checkMonitor(m, clk.Now())
|
||||
clk.Advance(15 * time.Second)
|
||||
}
|
||||
if s.monitorState("hb").Status != "down" {
|
||||
t.Fatalf("missed heartbeat should be down: %+v", s.monitorState("hb"))
|
||||
}
|
||||
if !s.Heartbeat(m.Token) {
|
||||
t.Fatal("ping rejected")
|
||||
}
|
||||
if s.monitorState("hb").Status != "up" || len(s.listAlerts("open", 10)) != 0 {
|
||||
t.Fatal("a ping should recover a down heartbeat immediately")
|
||||
}
|
||||
if s.Heartbeat("nope") {
|
||||
t.Fatal("unknown token accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncidentLifecycle(t *testing.T) {
|
||||
s, _, _ := newTestServer(t)
|
||||
u := mustUser(t, s, "Alice")
|
||||
a, _, _ := s.RaiseAlert("k", "t", "api 500s", "", "critical", "")
|
||||
in, err := s.DeclareIncident("API outage", "sev1", "errors everywhere", u.ID, u.Name, []string{a.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := s.getAlert(a.ID); got.IncidentID != in.ID {
|
||||
t.Fatal("alert not linked")
|
||||
}
|
||||
st := "identified"
|
||||
if _, err := s.UpdateIncident(in.ID, u.Name, IncidentPatch{Status: &st, Note: "bad deploy"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bogus := "exploding"
|
||||
if _, err := s.UpdateIncident(in.ID, u.Name, IncidentPatch{Status: &bogus}); err == nil {
|
||||
t.Fatal("invalid status accepted")
|
||||
}
|
||||
done := "resolved"
|
||||
got, _ := s.UpdateIncident(in.ID, u.Name, IncidentPatch{Status: &done})
|
||||
if got.Resolved == 0 {
|
||||
t.Fatal("resolved time not set")
|
||||
}
|
||||
s.CommentIncident(in.ID, u.Name, "rolled back")
|
||||
ev := s.st.events("incident", in.ID)
|
||||
if len(ev) < 4 {
|
||||
t.Fatalf("timeline too short: %+v", ev)
|
||||
}
|
||||
if tpl := s.postmortemTemplate(got); !strings.Contains(tpl, "rolled back") || !strings.Contains(tpl, "API outage") {
|
||||
t.Fatalf("template missing content:\n%s", tpl)
|
||||
}
|
||||
if n := len(s.MessagesSince("incidents", 0, 10)); n != 3 {
|
||||
t.Fatalf("expected declare + 2 updates on the incidents topic, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelSecretsAreSealedAndRedacted(t *testing.T) {
|
||||
s, _, _ := newTestServer(t)
|
||||
ts := httptest.NewServer(s.Handler())
|
||||
defer ts.Close()
|
||||
c := newAPIClient(t, ts)
|
||||
c.setup()
|
||||
var created map[string]any
|
||||
c.do("POST", "/api/v1/channels", map[string]any{"name": "slack", "kind": "webhook", "target": "https://hooks.example.com/secret-token", "format": "slack", "enabled": true}, &created)
|
||||
var raw string
|
||||
s.st.db.QueryRow(`SELECT body FROM docs WHERE kind='channel'`).Scan(&raw)
|
||||
if strings.Contains(raw, "secret-token") {
|
||||
t.Fatal("webhook URL stored in clear text")
|
||||
}
|
||||
if created["target"] == "https://hooks.example.com/secret-token" {
|
||||
t.Fatal("the API returned the secret")
|
||||
}
|
||||
// Editing without touching the target keeps the stored secret.
|
||||
created["name"] = "slack2"
|
||||
c.do("PUT", "/api/v1/channels/"+created["id"].(string), created, nil)
|
||||
ch, _ := getOf[Channel](s.st, "channel", created["id"].(string))
|
||||
if s.st.open(ch.Target) != "https://hooks.example.com/secret-token" || ch.Name != "slack2" {
|
||||
t.Fatalf("secret lost on edit: %+v", ch)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- HTTP API end to end ----------
|
||||
|
||||
type apiClient struct {
|
||||
t *testing.T
|
||||
ts *httptest.Server
|
||||
cookie *http.Cookie
|
||||
}
|
||||
|
||||
func newAPIClient(t *testing.T, ts *httptest.Server) *apiClient { return &apiClient{t: t, ts: ts} }
|
||||
|
||||
func (c *apiClient) req(method, path string, body any, headers map[string]string) *http.Response {
|
||||
c.t.Helper()
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rd = strings.NewReader(string(b))
|
||||
}
|
||||
r, _ := http.NewRequest(method, c.ts.URL+path, rd)
|
||||
if body != nil || method != "GET" {
|
||||
r.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if c.cookie != nil {
|
||||
r.AddCookie(c.cookie)
|
||||
}
|
||||
for k, v := range headers {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(r)
|
||||
if err != nil {
|
||||
c.t.Fatal(err)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func (c *apiClient) do(method, path string, body any, out any) int {
|
||||
c.t.Helper()
|
||||
resp := c.req(method, path, body, nil)
|
||||
defer resp.Body.Close()
|
||||
if out != nil {
|
||||
json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
func (c *apiClient) setup() {
|
||||
c.t.Helper()
|
||||
resp := c.req("POST", "/api/v1/setup", map[string]string{"name": "Root", "email": "root@example.com", "password": "correct-horse-battery"}, nil)
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
c.t.Fatalf("setup: %d", resp.StatusCode)
|
||||
}
|
||||
for _, ck := range resp.Cookies() {
|
||||
if ck.Name == sessionCookie {
|
||||
c.cookie = ck
|
||||
}
|
||||
}
|
||||
if c.cookie == nil {
|
||||
c.t.Fatal("no session cookie")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIAuthSetupAndTopics(t *testing.T) {
|
||||
s, _, _ := newTestServer(t)
|
||||
ts := httptest.NewServer(s.Handler())
|
||||
defer ts.Close()
|
||||
c := newAPIClient(t, ts)
|
||||
|
||||
if code := c.do("GET", "/api/v1/alerts", nil, nil); code != 401 {
|
||||
t.Fatalf("unauthenticated API call: %d", code)
|
||||
}
|
||||
if code := c.do("POST", "/mytopic", nil, nil); code != 401 {
|
||||
t.Fatalf("unauthenticated publish: %d", code)
|
||||
}
|
||||
c.setup()
|
||||
if code := c.do("POST", "/api/v1/setup", map[string]string{"name": "x", "email": "x@x.com", "password": "correct-horse-battery"}, nil); code != 403 {
|
||||
t.Fatalf("second setup must be refused, got %d", code)
|
||||
}
|
||||
|
||||
var key map[string]string
|
||||
c.do("POST", "/api/v1/apikeys", map[string]string{"name": "ci"}, &key)
|
||||
if !strings.HasPrefix(key["token"], "sb_") {
|
||||
t.Fatalf("api key: %v", key)
|
||||
}
|
||||
bearer := map[string]string{"Authorization": "Bearer " + key["token"]}
|
||||
|
||||
// Subscribe with the API key, then publish ntfy-style with headers.
|
||||
sub := make(chan string, 4)
|
||||
go func() {
|
||||
req, _ := http.NewRequest("GET", ts.URL+"/alerts-room/sse", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+key["token"])
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
sc := bufio.NewScanner(resp.Body)
|
||||
for sc.Scan() {
|
||||
if strings.HasPrefix(sc.Text(), "data: ") {
|
||||
sub <- sc.Text()[6:]
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
r, _ := http.NewRequest("POST", ts.URL+"/alerts-room", strings.NewReader("disk is full"))
|
||||
r.Header.Set("Authorization", "Bearer "+key["token"])
|
||||
r.Header.Set("Title", "Disk")
|
||||
r.Header.Set("Priority", "urgent")
|
||||
r.Header.Set("Tags", "warning,disk")
|
||||
resp, err := http.DefaultClient.Do(r)
|
||||
if err != nil || resp.StatusCode != 200 {
|
||||
t.Fatalf("publish: %v %v", err, resp)
|
||||
}
|
||||
select {
|
||||
case line := <-sub:
|
||||
var m Message
|
||||
json.Unmarshal([]byte(line), &m)
|
||||
if m.Message != "disk is full" || m.Title != "Disk" || m.Priority != 5 || len(m.Tags) != 2 {
|
||||
t.Fatalf("bad message: %+v", m)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("subscriber got nothing")
|
||||
}
|
||||
|
||||
// Reserved topics are refused.
|
||||
for _, reserved := range []string{"/api", "/status"} {
|
||||
r, _ := http.NewRequest("POST", ts.URL+reserved, strings.NewReader("x"))
|
||||
r.Header.Set("Authorization", "Bearer "+key["token"])
|
||||
resp, _ := http.DefaultClient.Do(r)
|
||||
if resp.StatusCode == 200 {
|
||||
t.Errorf("publish to %s should not succeed", reserved)
|
||||
}
|
||||
}
|
||||
|
||||
// Inbound alert -> ack -> resolve over the API with the key.
|
||||
var out map[string]any
|
||||
c.cookie = nil
|
||||
if code := c.do("POST", "/api/v1/alerts", map[string]string{"key": "cpu", "summary": "CPU high"}, nil); code != 401 {
|
||||
t.Fatalf("alert without credentials: %d", code)
|
||||
}
|
||||
resp = c.req("POST", "/api/v1/alerts", map[string]string{"key": "cpu", "summary": "CPU high", "severity": "warning"}, bearer)
|
||||
json.NewDecoder(resp.Body).Decode(&out)
|
||||
resp.Body.Close()
|
||||
id, _ := out["id"].(string)
|
||||
if id == "" {
|
||||
t.Fatalf("no alert id: %v", out)
|
||||
}
|
||||
c.req("POST", "/api/v1/alerts/"+id+"/ack", nil, bearer).Body.Close()
|
||||
a, _ := s.getAlert(id)
|
||||
if a.Status != "acked" || a.AckedBy != "Root" {
|
||||
t.Fatalf("ack: %+v", a)
|
||||
}
|
||||
c.req("POST", "/api/v1/alerts", map[string]string{"key": "cpu", "status": "resolved"}, bearer).Body.Close()
|
||||
if a, _ = s.getAlert(id); a.Status != "resolved" {
|
||||
t.Fatalf("resolve via key: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertmanagerWebhook(t *testing.T) {
|
||||
s, _, _ := newTestServer(t)
|
||||
ts := httptest.NewServer(s.Handler())
|
||||
defer ts.Close()
|
||||
c := newAPIClient(t, ts)
|
||||
c.setup()
|
||||
payload := map[string]any{"alerts": []map[string]any{{"status": "firing", "fingerprint": "abc",
|
||||
"labels": map[string]string{"alertname": "HighLatency", "severity": "critical"}, "annotations": map[string]string{"summary": "p99 > 2s"}}}}
|
||||
c.do("POST", "/api/v1/integrations/alertmanager", payload, nil)
|
||||
open := s.listAlerts("open", 10)
|
||||
if len(open) != 1 || open[0].Summary != "p99 > 2s" || open[0].Severity != "critical" {
|
||||
t.Fatalf("got %+v", open)
|
||||
}
|
||||
payload["alerts"].([]map[string]any)[0]["status"] = "resolved"
|
||||
c.do("POST", "/api/v1/integrations/alertmanager", payload, nil)
|
||||
if len(s.listAlerts("open", 10)) != 0 {
|
||||
t.Fatal("not resolved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusPageOnlyShowsPublic(t *testing.T) {
|
||||
s, clk, _ := newTestServer(t)
|
||||
s.st.put("monitor", "a", &Monitor{ID: "a", Name: "Public site", Kind: "http", Target: "http://x", Enabled: true, Public: true})
|
||||
s.st.put("monitor", "b", &Monitor{ID: "b", Name: "Internal DB", Kind: "tcp", Target: "x:1", Enabled: true})
|
||||
s.saveState(MonitorState{MonitorID: "a", Status: "up", Since: clk.Now().Unix()})
|
||||
s.st.put("incident", "i1", &Incident{ID: "i1", Title: "Public incident", Severity: "sev2", Status: "investigating", Public: true, Created: clk.Now().Unix()})
|
||||
s.st.put("incident", "i2", &Incident{ID: "i2", Title: "Private incident", Severity: "sev1", Status: "investigating", Created: clk.Now().Unix()})
|
||||
ts := httptest.NewServer(s.Handler())
|
||||
defer ts.Close()
|
||||
resp, _ := http.Get(ts.URL + "/status.json")
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
body := string(b)
|
||||
if !strings.Contains(body, "Public site") || strings.Contains(body, "Internal DB") {
|
||||
t.Fatalf("monitor visibility wrong: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, "Public incident") || strings.Contains(body, "Private incident") {
|
||||
t.Fatalf("incident visibility wrong: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, `"overall":"degraded"`) {
|
||||
t.Fatalf("overall: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginThrottle(t *testing.T) {
|
||||
s, _, _ := newTestServer(t)
|
||||
mustUser(t, s, "Alice")
|
||||
ts := httptest.NewServer(s.Handler())
|
||||
defer ts.Close()
|
||||
c := newAPIClient(t, ts)
|
||||
var last int
|
||||
for i := 0; i < 12; i++ {
|
||||
last = c.do("POST", "/api/v1/login", map[string]string{"email": "alice@example.com", "password": "wrong-password-x"}, nil)
|
||||
}
|
||||
if last != 429 {
|
||||
t.Fatalf("expected throttling after repeated failures, got %d", last)
|
||||
}
|
||||
// Even the right password is refused while throttled.
|
||||
if code := c.do("POST", "/api/v1/login", map[string]string{"email": "alice@example.com", "password": "correct-horse-battery"}, nil); code != 429 {
|
||||
t.Fatalf("throttle should apply to the right password too, got %d", code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
//go:embed web/index.html
|
||||
var indexHTML []byte
|
||||
|
||||
//go:embed web/status.html
|
||||
var statusHTML []byte
|
||||
|
||||
type SMTPConfig struct{ Host, Port, User, Password, From string }
|
||||
|
||||
// Options configures a Server. Zero values are sensible defaults.
|
||||
type Options struct {
|
||||
DataDir string
|
||||
URL string // public URL, used for links in notifications
|
||||
MasterKey string
|
||||
MetricsToken string
|
||||
SMTP SMTPConfig
|
||||
MessageDays int // how long topic messages are kept (default 30)
|
||||
CheckDays int // how long monitor results are kept (default 30)
|
||||
|
||||
// Tests replace these.
|
||||
Now func() time.Time
|
||||
HTTPClient *http.Client
|
||||
Mailer func(to, subject, body string) error
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
opts Options
|
||||
st *Store
|
||||
hub *hub
|
||||
|
||||
// monitor runtime state
|
||||
mu sync.Mutex
|
||||
running map[string]bool // monitor ids with a check in flight
|
||||
|
||||
loginMu sync.Mutex
|
||||
loginTry map[string][]time.Time
|
||||
|
||||
stop chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func New(opts Options, st *Store) *Server {
|
||||
if opts.MessageDays <= 0 {
|
||||
opts.MessageDays = 30
|
||||
}
|
||||
if opts.CheckDays <= 0 {
|
||||
opts.CheckDays = 30
|
||||
}
|
||||
return &Server{opts: opts, st: st, hub: newHub(), running: map[string]bool{}, loginTry: map[string][]time.Time{}, stop: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (s *Server) now() time.Time {
|
||||
if s.opts.Now != nil {
|
||||
return s.opts.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// Start runs the background workers: monitor scheduler, escalation, pruning.
|
||||
func (s *Server) Start() {
|
||||
s.every(time.Second, func() { s.runDueMonitors(s.now()) })
|
||||
s.every(5*time.Second, func() { s.escalate(s.now()) })
|
||||
s.every(time.Hour, func() { s.st.prune(s.now(), s.opts.MessageDays, s.opts.CheckDays) })
|
||||
}
|
||||
|
||||
func (s *Server) Close() {
|
||||
close(s.stop)
|
||||
s.wg.Wait()
|
||||
}
|
||||
|
||||
func (s *Server) every(d time.Duration, f func()) {
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
t := time.NewTicker(d)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-s.stop:
|
||||
return
|
||||
case <-t.C:
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("worker panic: %v", r)
|
||||
}
|
||||
}()
|
||||
f()
|
||||
}()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Server) link(path string) string {
|
||||
if s.opts.URL == "" {
|
||||
return ""
|
||||
}
|
||||
return s.opts.URL + path
|
||||
}
|
||||
|
||||
// BootstrapAdmin creates the first administrator from the environment when there are no users yet.
|
||||
func (s *Server) BootstrapAdmin(email, password string) {
|
||||
if email == "" || password == "" || s.countUsers() > 0 {
|
||||
return
|
||||
}
|
||||
if _, err := s.createUser("Admin", email, password, "admin", ""); err != nil {
|
||||
log.Printf("SANDBAR_ADMIN_EMAIL: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.veillax.com/Veillax/reef/secretbox"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Store is Sandbar's one SQLite file (pure Go, WAL mode). Configuration that is
|
||||
// edited by hand and rarely queried (channels, monitors, schedules, policies,
|
||||
// incidents) is kept as JSON documents in one table; things that grow or are
|
||||
// searched (messages, checks, alerts, timeline events) have proper indexed tables.
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
box *secretbox.Box
|
||||
}
|
||||
|
||||
var errNotFound = errors.New("not found")
|
||||
|
||||
const schema = `
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL, email TEXT NOT NULL UNIQUE COLLATE NOCASE,
|
||||
hash TEXT NOT NULL, role TEXT NOT NULL, phone TEXT NOT NULL DEFAULT '', created INTEGER NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
hash TEXT PRIMARY KEY, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, expires INTEGER NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS apikeys (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL, hash TEXT NOT NULL UNIQUE,
|
||||
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, created INTEGER NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS docs (
|
||||
kind TEXT NOT NULL, id TEXT NOT NULL, body TEXT NOT NULL, PRIMARY KEY (kind, id));
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id TEXT PRIMARY KEY, topic TEXT NOT NULL, time INTEGER NOT NULL, title TEXT NOT NULL,
|
||||
body TEXT NOT NULL, priority INTEGER NOT NULL, tags TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS messages_topic ON messages(topic, time);
|
||||
CREATE TABLE IF NOT EXISTS checks (
|
||||
monitor_id TEXT NOT NULL, time INTEGER NOT NULL, ok INTEGER NOT NULL, ms INTEGER NOT NULL, detail TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS checks_monitor ON checks(monitor_id, time);
|
||||
CREATE TABLE IF NOT EXISTS mstate (
|
||||
monitor_id TEXT PRIMARY KEY, status TEXT NOT NULL, since INTEGER NOT NULL, last_check INTEGER NOT NULL,
|
||||
last_ms INTEGER NOT NULL, fails INTEGER NOT NULL, last_ping INTEGER NOT NULL, detail TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS alerts (
|
||||
id TEXT PRIMARY KEY, key TEXT NOT NULL, source TEXT NOT NULL, summary TEXT NOT NULL, detail TEXT NOT NULL,
|
||||
severity TEXT NOT NULL, status TEXT NOT NULL, policy_id TEXT NOT NULL, step INTEGER NOT NULL,
|
||||
next_at INTEGER NOT NULL, created INTEGER NOT NULL, updated INTEGER NOT NULL, count INTEGER NOT NULL,
|
||||
acked_by TEXT NOT NULL DEFAULT '', acked_at INTEGER NOT NULL DEFAULT 0, snooze_until INTEGER NOT NULL DEFAULT 0,
|
||||
resolved_at INTEGER NOT NULL DEFAULT 0, incident_id TEXT NOT NULL DEFAULT '');
|
||||
CREATE INDEX IF NOT EXISTS alerts_key ON alerts(key, status);
|
||||
CREATE INDEX IF NOT EXISTS alerts_due ON alerts(status, next_at);
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, scope TEXT NOT NULL, ref TEXT NOT NULL, time INTEGER NOT NULL,
|
||||
kind TEXT NOT NULL, actor TEXT NOT NULL, text TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS events_ref ON events(scope, ref, time);
|
||||
`
|
||||
|
||||
// OpenStore opens (creating if needed) the database in dir.
|
||||
func OpenStore(dir string, box *secretbox.Box) (*Store, error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "sandbar.db")) +
|
||||
"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)&_pragma=synchronous(NORMAL)"
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(4)
|
||||
if _, err := db.Exec(schema); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &Store{db: db, box: box}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
// ---------- ids and tokens ----------
|
||||
|
||||
func newID() string {
|
||||
b := make([]byte, 9)
|
||||
rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
func newToken() string {
|
||||
b := make([]byte, 32)
|
||||
rand.Read(b)
|
||||
return base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
func hashToken(t string) string {
|
||||
sum := sha256.Sum256([]byte(t))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// ---------- JSON documents ----------
|
||||
|
||||
func (s *Store) put(kind, id string, v any) error {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(`INSERT INTO docs(kind,id,body) VALUES(?,?,?) ON CONFLICT(kind,id) DO UPDATE SET body=excluded.body`, kind, id, string(b))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) get(kind, id string, v any) error {
|
||||
var body string
|
||||
err := s.db.QueryRow(`SELECT body FROM docs WHERE kind=? AND id=?`, kind, id).Scan(&body)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return errNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal([]byte(body), v)
|
||||
}
|
||||
|
||||
// list decodes every document of a kind through add, which is called with a
|
||||
// fresh pointer-to-value from newItem and then appended by the caller.
|
||||
func (s *Store) list(kind string, each func(raw []byte) error) error {
|
||||
rows, err := s.db.Query(`SELECT body FROM docs WHERE kind=? ORDER BY id`, kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var bodies []string
|
||||
for rows.Next() {
|
||||
var b string
|
||||
if err := rows.Scan(&b); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
bodies = append(bodies, b)
|
||||
}
|
||||
rows.Close()
|
||||
for _, b := range bodies {
|
||||
if err := each([]byte(b)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) del(kind, id string) error {
|
||||
res, err := s.db.Exec(`DELETE FROM docs WHERE kind=? AND id=?`, kind, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
return errNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func listOf[T any](s *Store, kind string) ([]T, error) {
|
||||
out := []T{}
|
||||
err := s.list(kind, func(raw []byte) error {
|
||||
var v T
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, v)
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func getOf[T any](s *Store, kind, id string) (*T, error) {
|
||||
var v T
|
||||
if err := s.get(kind, id, &v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
// seal and open wrap secretbox so callers don't repeat the nil check.
|
||||
func (s *Store) seal(v string) string {
|
||||
if s.box == nil || v == "" {
|
||||
return v
|
||||
}
|
||||
return s.box.Seal(v)
|
||||
}
|
||||
|
||||
func (s *Store) open(v string) string {
|
||||
if s.box == nil {
|
||||
return v
|
||||
}
|
||||
out, err := s.box.Open(v)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------- timeline events ----------
|
||||
|
||||
func (s *Store) addEvent(scope, ref, kind, actor, text string, at time.Time) {
|
||||
s.db.Exec(`INSERT INTO events(scope,ref,time,kind,actor,text) VALUES(?,?,?,?,?,?)`, scope, ref, at.Unix(), kind, actor, text)
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
Time int64 `json:"time"`
|
||||
Kind string `json:"kind"`
|
||||
Actor string `json:"actor"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
func (s *Store) events(scope, ref string) []Event {
|
||||
out := []Event{}
|
||||
rows, err := s.db.Query(`SELECT time,kind,actor,text FROM events WHERE scope=? AND ref=? ORDER BY time, id`, scope, ref)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var e Event
|
||||
if rows.Scan(&e.Time, &e.Kind, &e.Actor, &e.Text) == nil {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// prune deletes data past its retention.
|
||||
func (s *Store) prune(now time.Time, messageDays, checkDays int) {
|
||||
s.db.Exec(`DELETE FROM messages WHERE time < ?`, now.AddDate(0, 0, -messageDays).Unix())
|
||||
s.db.Exec(`DELETE FROM checks WHERE time < ?`, now.AddDate(0, 0, -checkDays).Unix())
|
||||
s.db.Exec(`DELETE FROM sessions WHERE expires < ?`, now.Unix())
|
||||
s.db.Exec(`DELETE FROM alerts WHERE status='resolved' AND resolved_at < ? AND incident_id=''`, now.AddDate(0, 0, -90).Unix())
|
||||
}
|
||||
@@ -0,0 +1,775 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="color-scheme" content="light dark">
|
||||
<title>Sandbar</title>
|
||||
<style>
|
||||
/* tokens:begin Design tokens, the Lightwell design language (docs/design.md). This is the only place raw colors and sizes are defined. */
|
||||
:root {
|
||||
--accent:#1f5c5c; --accent-contrast:#ffffff;
|
||||
--bg:#f6f5f3; --surface:#ffffff; --surface-2:#efede9; --border:#e0ddd7; --border-strong:#c4c0b8;
|
||||
--text:#1d1c1a; --text-muted:#5f5b54; --text-faint:#7a766e;
|
||||
--accent-text:var(--accent);
|
||||
--accent-soft:color-mix(in srgb, var(--accent) 9%, var(--surface));
|
||||
--accent-hover:color-mix(in srgb, var(--accent) 88%, #000000);
|
||||
--focus:color-mix(in srgb, var(--accent) 70%, #ffffff);
|
||||
--success:#1c6b3a; --success-bg:#e3f1e8; --warning:#85560a; --warning-bg:#f9eed6; --danger:#a3271e; --danger-bg:#f9e6e3; --neutral-bg:#eceae6;
|
||||
--overlay:rgb(20 19 17 / 0.55); --shadow:0 8px 24px rgb(20 19 17 / 0.14);
|
||||
--s1:4px; --s2:8px; --s3:12px; --s4:16px; --s5:24px; --s6:32px; --s7:48px; --s8:64px;
|
||||
--radius:6px; --radius-lg:10px; --border-w:1px;
|
||||
--font:system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;
|
||||
--font-mono:ui-monospace,"SF Mono",Menlo,Consolas,monospace;
|
||||
--text-xs:12px; --text-sm:13px; --text-base:15px; --text-lg:18px; --text-xl:22px; --text-2xl:28px; --leading:1.5;
|
||||
--sidebar-w:232px; --content-max:1120px; --touch:44px; --control-h:38px;
|
||||
--z-sticky:20; --z-overlay:100; --z-toast:200; --ease:140ms ease;
|
||||
color-scheme:light;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) { :root:not([data-theme="light"]) {
|
||||
--bg:#131312; --surface:#1b1b19; --surface-2:#252523; --border:#33322e; --border-strong:#4a4943;
|
||||
--text:#ecebe7; --text-muted:#adaaa2; --text-faint:#8f8c84;
|
||||
--accent-text:color-mix(in srgb, var(--accent) 45%, #ffffff);
|
||||
--accent-soft:color-mix(in srgb, var(--accent) 22%, var(--surface));
|
||||
--accent-hover:color-mix(in srgb, var(--accent) 85%, #ffffff);
|
||||
--success:#86d3a0; --success-bg:#17301f; --warning:#e8bd6b; --warning-bg:#33280f; --danger:#f19a92; --danger-bg:#3a1b18; --neutral-bg:#2b2a27;
|
||||
--overlay:rgb(0 0 0 / 0.65); --shadow:0 8px 24px rgb(0 0 0 / 0.5); color-scheme:dark;
|
||||
} }
|
||||
:root[data-theme="dark"] {
|
||||
--bg:#131312; --surface:#1b1b19; --surface-2:#252523; --border:#33322e; --border-strong:#4a4943;
|
||||
--text:#ecebe7; --text-muted:#adaaa2; --text-faint:#8f8c84;
|
||||
--accent-text:color-mix(in srgb, var(--accent) 45%, #ffffff);
|
||||
--accent-soft:color-mix(in srgb, var(--accent) 22%, var(--surface));
|
||||
--accent-hover:color-mix(in srgb, var(--accent) 85%, #ffffff);
|
||||
--success:#86d3a0; --success-bg:#17301f; --warning:#e8bd6b; --warning-bg:#33280f; --danger:#f19a92; --danger-bg:#3a1b18; --neutral-bg:#2b2a27;
|
||||
--overlay:rgb(0 0 0 / 0.65); --shadow:0 8px 24px rgb(0 0 0 / 0.5); color-scheme:dark;
|
||||
}
|
||||
/* tokens:end */
|
||||
|
||||
*, *::before, *::after { box-sizing:border-box; }
|
||||
html, body { height:100%; }
|
||||
html { -webkit-text-size-adjust:100%; }
|
||||
body { margin:0; background:var(--bg); color:var(--text); font:var(--text-base)/var(--leading) var(--font); -webkit-font-smoothing:antialiased; }
|
||||
h1, h2, h3, h4 { margin:0; line-height:1.25; font-weight:600; letter-spacing:-.01em; }
|
||||
h1 { font-size:var(--text-2xl); } h2 { font-size:var(--text-xl); } h3 { font-size:var(--text-lg); }
|
||||
p { margin:0 0 var(--s3); }
|
||||
a { color:var(--accent-text); text-underline-offset:2px; }
|
||||
a:hover { text-decoration-thickness:2px; }
|
||||
:focus-visible { outline:2px solid var(--focus); outline-offset:2px; border-radius:var(--radius); }
|
||||
.sr-only { position:absolute; width:1px; height:1px; margin:-1px; padding:0; overflow:hidden; clip:rect(0 0 0 0); white-space:nowrap; border:0; }
|
||||
.skip { position:absolute; left:var(--s3); top:-100px; z-index:var(--z-toast); background:var(--surface); border:var(--border-w) solid var(--border-strong); border-radius:var(--radius); padding:var(--s2) var(--s3); }
|
||||
.skip:focus { top:var(--s3); }
|
||||
.dim { color:var(--text-muted); } .err { color:var(--danger); } .small { font-size:var(--text-sm); }
|
||||
.num { font-variant-numeric:tabular-nums; }
|
||||
/* long machine messages (certificate errors) wrap and stop at three lines; the full text is in the tooltip */
|
||||
.clip { display:-webkit-box; -webkit-line-clamp:3; -webkit-box-orient:vertical; overflow:hidden; overflow-wrap:anywhere; max-width:340px; }
|
||||
code, .mono { font-family:var(--font-mono); font-size:var(--text-sm); }
|
||||
.icon { width:18px; height:18px; flex:none; stroke-width:1.75; }
|
||||
|
||||
/* shell */
|
||||
.shell { display:grid; grid-template-columns:var(--sidebar-w) minmax(0,1fr); min-height:100vh; }
|
||||
.sidebar { position:sticky; top:0; height:100vh; display:flex; flex-direction:column; gap:var(--s3); padding:var(--s5) var(--s3); background:var(--surface); border-right:var(--border-w) solid var(--border); overflow-y:auto; }
|
||||
.sidebar-head { display:flex; align-items:center; justify-content:space-between; gap:var(--s2); }
|
||||
.sidebar button.sidebar-close { display:none; }
|
||||
.brand { display:flex; align-items:center; gap:var(--s2); font-size:var(--text-lg); font-weight:700; letter-spacing:-.02em; padding:0 var(--s3); color:var(--text); text-decoration:none; }
|
||||
.brand .icon { color:var(--accent-text); }
|
||||
.orgsel select { width:100%; }
|
||||
.navsec { font-size:var(--text-xs); font-weight:600; letter-spacing:.06em; text-transform:uppercase; color:var(--text-muted); padding:var(--s4) var(--s3) var(--s1); }
|
||||
.nav { display:flex; flex-direction:column; gap:2px; }
|
||||
.navitem { display:flex; align-items:center; gap:var(--s3); width:100%; min-height:36px; padding:var(--s2) var(--s3); border-radius:var(--radius); color:var(--text-muted); text-decoration:none; font:inherit; font-weight:500; background:none; border:0; cursor:pointer; text-align:left; }
|
||||
.navitem:hover { background:var(--surface-2); color:var(--text); }
|
||||
.navitem.on { background:var(--accent-soft); color:var(--accent-text); }
|
||||
.grow { flex:1 1 auto; min-width:0; }
|
||||
.userbox { border-top:var(--border-w) solid var(--border); padding-top:var(--s3); }
|
||||
.userbox .who { padding:var(--s1) var(--s3) var(--s2); font-size:var(--text-sm); color:var(--text-muted); overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
|
||||
.content { min-width:0; }
|
||||
main { max-width:var(--content-max); margin:0 auto; padding:var(--s6) var(--s5) var(--s8); }
|
||||
.topbar, .backdrop { display:none; }
|
||||
.page-h { display:flex; align-items:flex-start; justify-content:space-between; gap:var(--s4); margin-bottom:var(--s5); flex-wrap:wrap; }
|
||||
.page-h p { margin:var(--s1) 0 0; color:var(--text-muted); max-width:70ch; }
|
||||
.page-h .btn { white-space:nowrap; }
|
||||
|
||||
/* cards, layout */
|
||||
.card { background:var(--surface); border:var(--border-w) solid var(--border); border-radius:var(--radius-lg); padding:var(--s5); margin-bottom:var(--s4); overflow-x:auto; }
|
||||
.card.flush { padding:0; overflow:hidden; }
|
||||
.card > h2 { font-size:var(--text-lg); margin:0 0 var(--s3); }
|
||||
.row-h { display:flex; align-items:center; justify-content:flex-end; gap:var(--s3); margin-bottom:var(--s3); flex-wrap:wrap; }
|
||||
.row-h h2 { display:none; }
|
||||
.grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(170px,1fr)); gap:var(--s4); align-items:start; }
|
||||
.stack { display:flex; flex-direction:column; gap:var(--s4); }
|
||||
/* a button or checkbox in a row of fields lines up with the inputs, below their labels */
|
||||
.grid > button { justify-self:start; margin-top:calc(var(--text-sm) * var(--leading) + var(--s1)); }
|
||||
.grid > label.chk { margin-top:calc(var(--text-sm) * var(--leading) + var(--s1) + var(--s2)); }
|
||||
.sect-h { padding:var(--s3) var(--s4); font-weight:600; border-bottom:var(--border-w) solid var(--border); }
|
||||
.scroll { overflow-x:auto; }
|
||||
.actions { display:flex; gap:var(--s2); flex-wrap:wrap; }
|
||||
.form-actions { justify-content:flex-end; margin-top:var(--s2); }
|
||||
.toolbar { display:flex; align-items:center; gap:var(--s3); margin-bottom:var(--s3); flex-wrap:wrap; }
|
||||
.toolbar input[type=search] { max-width:320px; }
|
||||
.center { max-width:440px; margin:var(--s8) auto; padding:0 var(--s4); }
|
||||
.center .brand { justify-content:center; font-size:var(--text-xl); padding-bottom:var(--s4); }
|
||||
|
||||
/* choices (add resource) */
|
||||
.choices { display:grid; grid-template-columns:repeat(auto-fill,minmax(210px,1fr)); gap:var(--s3); }
|
||||
.choice { text-align:left; background:var(--surface); border:var(--border-w) solid var(--border-strong); border-radius:var(--radius-lg); padding:var(--s3) var(--s4); color:var(--text); cursor:pointer; font:inherit; display:flex; flex-direction:column; gap:var(--s1); }
|
||||
.choice:hover { background:var(--surface-2); }
|
||||
.choice.on { border-color:var(--accent); background:var(--accent-soft); }
|
||||
.choice-t { font-weight:600; }
|
||||
|
||||
/* update progress and log */
|
||||
.progress { height:6px; background:var(--surface-2); border-radius:999px; overflow:hidden; }
|
||||
.progress > div { height:100%; background:var(--accent); transition:width var(--ease); }
|
||||
.progress > div.bad { background:var(--danger); }
|
||||
.logbox { margin:0; background:var(--bg); border:var(--border-w) solid var(--border); border-radius:var(--radius); padding:var(--s3); font:var(--text-xs)/1.6 var(--font-mono); max-height:260px; overflow:auto; white-space:pre-wrap; word-break:break-word; }
|
||||
|
||||
/* forms */
|
||||
label { display:flex; flex-direction:column; gap:var(--s1); font-size:var(--text-sm); font-weight:600; color:var(--text); }
|
||||
label.chk { flex-direction:row; align-items:flex-start; gap:var(--s2); font-size:var(--text-base); font-weight:400; }
|
||||
input, select, textarea { background:var(--surface); color:var(--text); border:var(--border-w) solid var(--border-strong); border-radius:var(--radius); padding:var(--s2) var(--s3); min-height:var(--control-h); font:inherit; font-weight:400; min-width:0; width:100%; }
|
||||
textarea { min-height:96px; resize:vertical; }
|
||||
input[type=checkbox] { width:16px; height:16px; min-height:0; padding:0; margin-top:3px; accent-color:var(--accent); }
|
||||
input:focus, select:focus, textarea:focus { outline:2px solid var(--focus); outline-offset:0; border-color:var(--accent); }
|
||||
input:disabled, select:disabled { opacity:.6; }
|
||||
input[aria-invalid="true"] { border-color:var(--danger); }
|
||||
.sub { color:var(--text-muted); font-size:var(--text-sm); font-weight:400; }
|
||||
|
||||
/* buttons: one height and shape. btn is the primary action, ghost the default, danger and quiet the others */
|
||||
button.btn, button.ghost, .btn-link { display:inline-flex; align-items:center; justify-content:center; gap:var(--s2); min-height:var(--control-h); padding:0 var(--s4); border-radius:var(--radius); font:inherit; font-weight:500; cursor:pointer; white-space:nowrap; text-decoration:none; transition:background var(--ease), border-color var(--ease); }
|
||||
button.btn { background:var(--accent); color:var(--accent-contrast); border:var(--border-w) solid var(--accent); }
|
||||
button.btn:hover { background:var(--accent-hover); border-color:var(--accent-hover); }
|
||||
button.ghost { background:var(--surface); border:var(--border-w) solid var(--border-strong); color:var(--text); }
|
||||
button.ghost:hover { background:var(--surface-2); }
|
||||
button.quiet { background:transparent; border-color:transparent; }
|
||||
button.danger { color:var(--danger); border-color:var(--danger); }
|
||||
button.danger:hover { background:var(--danger-bg); }
|
||||
button.btn-icon { width:var(--control-h); padding:0; }
|
||||
td button.ghost, .actions button.ghost, .tgt button.ghost { min-height:30px; padding:0 var(--s3); font-size:var(--text-sm); }
|
||||
button:disabled { opacity:.55; cursor:not-allowed; }
|
||||
|
||||
/* tables */
|
||||
table { width:100%; border-collapse:collapse; }
|
||||
th { text-align:left; font-size:var(--text-sm); color:var(--text-muted); font-weight:600; padding:var(--s3) var(--s4); border-bottom:var(--border-w) solid var(--border); white-space:nowrap; background:var(--surface); }
|
||||
td { padding:var(--s3) var(--s4); border-bottom:var(--border-w) solid var(--border); vertical-align:middle; }
|
||||
tbody tr:hover td { background:var(--surface-2); }
|
||||
tr:last-child td { border-bottom:0; }
|
||||
|
||||
/* badges and tags: status always carries a text label, never color alone */
|
||||
.badge { display:inline-flex; align-items:center; gap:var(--s1); padding:2px var(--s2); border-radius:999px; font-size:var(--text-xs); font-weight:600; background:var(--neutral-bg); color:var(--text-muted); white-space:nowrap; }
|
||||
.badge-success { background:var(--success-bg); color:var(--success); }
|
||||
.badge-warning { background:var(--warning-bg); color:var(--warning); }
|
||||
.badge-danger { background:var(--danger-bg); color:var(--danger); }
|
||||
.badge-accent { background:var(--accent-soft); color:var(--accent-text); }
|
||||
.tag { display:inline-block; padding:1px var(--s2); border:var(--border-w) solid var(--border); border-radius:var(--radius); font-size:var(--text-xs); color:var(--text-muted); }
|
||||
pre { background:var(--surface-2); border:var(--border-w) solid var(--border); border-radius:var(--radius); padding:var(--s3); overflow:auto; margin:var(--s2) 0 0; user-select:all; white-space:pre-wrap; word-break:break-all; }
|
||||
|
||||
/* empty state, notice, loading */
|
||||
.empty { text-align:center; padding:var(--s7) var(--s5); color:var(--text-muted); border:var(--border-w) dashed var(--border-strong); border-radius:var(--radius-lg); margin:var(--s3); }
|
||||
.empty h3 { color:var(--text); margin-bottom:var(--s1); }
|
||||
.empty p { margin:0 auto var(--s3); max-width:56ch; }
|
||||
.empty .icon { width:28px; height:28px; margin:0 auto var(--s3); color:var(--text-faint); display:block; }
|
||||
.card > .empty { margin:0; }
|
||||
.notice { padding:var(--s3) var(--s4); border-radius:var(--radius); border:var(--border-w) solid var(--border); background:var(--surface-2); font-size:var(--text-sm); margin-bottom:var(--s4); }
|
||||
.notice-danger { background:var(--danger-bg); border-color:var(--danger); color:var(--danger); }
|
||||
.notice-success { background:var(--success-bg); border-color:var(--success); color:var(--success); }
|
||||
.skeleton { background:linear-gradient(90deg, var(--surface-2), var(--border), var(--surface-2)); background-size:200% 100%; animation:shimmer 1.3s infinite; border-radius:var(--radius); }
|
||||
@keyframes shimmer { to { background-position:-200% 0; } }
|
||||
|
||||
/* dialogs, tabs, toasts */
|
||||
.shade { position:fixed; inset:0; background:var(--overlay); display:grid; place-items:start center; padding:var(--s7) var(--s4); overflow:auto; z-index:var(--z-overlay); }
|
||||
.modal { background:var(--surface); border-radius:var(--radius-lg); box-shadow:var(--shadow); padding:var(--s5); width:min(480px,100%); }
|
||||
.modal.wide { width:min(780px,100%); }
|
||||
.modal h2 { margin:0 0 var(--s3); font-size:var(--text-lg); }
|
||||
fieldset { border:var(--border-w) solid var(--border); border-radius:var(--radius-lg); padding:var(--s4); margin:0; min-width:0; }
|
||||
legend { font-size:var(--text-sm); font-weight:600; color:var(--text-muted); padding:0 var(--s2); }
|
||||
.tgt { display:grid; grid-template-columns:1.2fr 1.4fr .7fr auto auto; gap:var(--s2); align-items:end; }
|
||||
.pillrow { display:flex; flex-wrap:wrap; gap:var(--s2) var(--s4); }
|
||||
.tabs2 { display:inline-flex; margin-bottom:var(--s4); border:var(--border-w) solid var(--border-strong); border-radius:var(--radius); overflow:hidden; max-width:100%; flex-wrap:wrap; background:var(--surface); }
|
||||
.tabs2 button { background:var(--surface); border:0; color:var(--text-muted); padding:0 var(--s3); min-height:var(--control-h); cursor:pointer; font:inherit; font-weight:500; }
|
||||
.tabs2 button + button { border-left:var(--border-w) solid var(--border-strong); }
|
||||
.tabs2 button:hover { background:var(--surface-2); }
|
||||
.tabs2 button.on { background:var(--accent-soft); color:var(--accent-text); }
|
||||
.toasts { position:fixed; right:var(--s4); bottom:var(--s4); z-index:var(--z-toast); display:flex; flex-direction:column; gap:var(--s2); max-width:min(360px, calc(100vw - 32px)); }
|
||||
.toast { background:var(--text); color:var(--bg); padding:var(--s3) var(--s4); border-radius:var(--radius); box-shadow:var(--shadow); font-size:var(--text-sm); }
|
||||
.toast-error { background:var(--danger); color:var(--accent-contrast); }
|
||||
|
||||
@media (prefers-reduced-motion: reduce) { .skeleton { animation:none; } * { transition:none !important; } html { scroll-behavior:auto; } }
|
||||
@media (pointer: coarse) {
|
||||
button.btn, button.ghost, .navitem, input, select { min-height:var(--touch); }
|
||||
td button.ghost, .actions button.ghost, .tgt button.ghost { min-height:36px; }
|
||||
input, select, textarea { font-size:16px; }
|
||||
}
|
||||
@media (max-width:860px) {
|
||||
.shell { grid-template-columns:minmax(0,1fr); grid-template-rows:auto minmax(0,1fr); }
|
||||
.sidebar { display:none; }
|
||||
.sidebar.open { display:flex; position:fixed; inset:0 auto 0 0; width:min(280px,85vw); height:100dvh; z-index:var(--z-overlay); box-shadow:var(--shadow); }
|
||||
.sidebar button.sidebar-close { display:inline-flex; }
|
||||
.topbar { display:flex; align-items:center; justify-content:space-between; min-height:52px; padding:var(--s2) var(--s4); background:var(--surface); border-bottom:var(--border-w) solid var(--border); position:sticky; top:0; z-index:var(--z-sticky); }
|
||||
.backdrop.open { display:block; position:fixed; inset:0; background:var(--overlay); z-index:calc(var(--z-overlay) - 1); }
|
||||
main { padding:var(--s5) var(--s4) var(--s7); }
|
||||
.tgt { grid-template-columns:1fr 1fr; }
|
||||
}
|
||||
@media (max-width:640px) {
|
||||
.modal { padding:var(--s4); }
|
||||
.shade { padding:var(--s4) var(--s3); }
|
||||
.toasts { left:var(--s3); right:var(--s3); bottom:var(--s3); max-width:none; }
|
||||
.page-h > .btn { width:100%; }
|
||||
}
|
||||
|
||||
/* Sandbar additions: built only from the tokens above */
|
||||
.listrow { display:flex; gap:var(--s2); align-items:flex-end; padding-bottom:var(--s3); border-bottom:var(--border-w) solid var(--border); }
|
||||
.listrow > :first-child { flex:1 1 auto; min-width:0; }
|
||||
.timeline { list-style:none; margin:0; padding:0; }
|
||||
.timeline li { display:flex; gap:var(--s3); padding:var(--s2) 0; border-bottom:var(--border-w) solid var(--border); font-size:var(--text-sm); }
|
||||
.timeline li:last-child { border-bottom:0; }
|
||||
.timeline .when { color:var(--text-muted); flex:none; min-width:150px; }
|
||||
.steps { margin:0; padding-left:var(--s5); }
|
||||
.feed { display:flex; flex-direction:column; gap:var(--s2); max-height:360px; overflow:auto; }
|
||||
.feed .empty { margin:0; }
|
||||
.msg { border:var(--border-w) solid var(--border); border-radius:var(--radius); padding:var(--s2) var(--s3); display:flex; flex-direction:column; gap:var(--s1); }
|
||||
.msg .tag { margin-right:var(--s1); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
<script>
|
||||
// ---------- tiny toolkit ----------
|
||||
function h(tag, props, ...kids) {
|
||||
const el = document.createElement(tag);
|
||||
for (const [k, v] of Object.entries(props || {})) {
|
||||
if (k.startsWith('on')) el.addEventListener(k.slice(2), v);
|
||||
else if (k === 'class') el.className = v;
|
||||
else if (v === true) el.setAttribute(k, '');
|
||||
else if (v !== false && v != null) el.setAttribute(k, v);
|
||||
}
|
||||
for (const k of kids.flat(Infinity)) if (k != null && k !== false) el.append(k.nodeType ? k : document.createTextNode(String(k)));
|
||||
return el;
|
||||
}
|
||||
const $app = document.getElementById('app');
|
||||
// theme: follows the system unless the person picked one
|
||||
try { const t = localStorage.getItem('theme'); if (t) document.documentElement.dataset.theme = t; } catch (e) {}
|
||||
function toggleTheme() {
|
||||
const dark = document.documentElement.dataset.theme ? document.documentElement.dataset.theme === 'dark' : matchMedia('(prefers-color-scheme: dark)').matches;
|
||||
const next = dark ? 'light' : 'dark';
|
||||
document.documentElement.dataset.theme = next;
|
||||
try { localStorage.setItem('theme', next); } catch (e) {}
|
||||
}
|
||||
// short confirmations at the bottom right; errors stay longer and are announced as alerts
|
||||
let toastBox;
|
||||
function toast(msg, bad) {
|
||||
if (!toastBox) { toastBox = h('div', {class:'toasts', 'aria-live':'polite'}); document.body.append(toastBox); }
|
||||
const t = h('div', {class:'toast' + (bad ? ' toast-error' : ''), role:bad ? 'alert' : 'status'}, msg);
|
||||
toastBox.append(t);
|
||||
setTimeout(() => t.remove(), bad ? 7000 : 3000);
|
||||
}
|
||||
const SIGNED_OUT = 'Your session ended. Sign in again to continue.';
|
||||
const V1 = '/api/v1';
|
||||
async function api(method, url, body) {
|
||||
const r = await fetch(url, { method, headers: {'Content-Type':'application/json'}, body: body !== undefined ? JSON.stringify(body) : undefined });
|
||||
const j = await r.json().catch(() => ({}));
|
||||
if (r.status === 401 && S.me) { S.me = null; boot(); throw new Error(SIGNED_OUT); }
|
||||
if (!r.ok) throw new Error(j.error || r.statusText);
|
||||
return j;
|
||||
}
|
||||
// run an action; show errors as a toast instead of throwing into the void
|
||||
const act = fn => async (...a) => { try { await fn(...a); } catch (e) { toast(e.message, true); } };
|
||||
// dates with the month written out, as the design language asks ("Oct 21, 2026")
|
||||
const fmtDate = t => new Date(t * 1000).toLocaleDateString(undefined, {month:'short', day:'numeric', year:'numeric'});
|
||||
const fmtTime = t => new Date(t * 1000).toLocaleString(undefined, {month:'short', day:'numeric', year:'numeric', hour:'numeric', minute:'2-digit'});
|
||||
const ago = t => { if (!t) return 'never'; const s = Date.now() / 1000 - t; return s < 90 ? 'just now' : s < 5400 ? Math.round(s / 60) + ' min ago' : s < 129600 ? Math.round(s / 3600) + ' h ago' : Math.round(s / 86400) + ' d ago'; };
|
||||
const toUnix = v => v ? Math.floor(new Date(v).getTime() / 1000) : 0;
|
||||
const fromUnix = t => { if (!t) return ''; const d = new Date(t * 1000); d.setMinutes(d.getMinutes() - d.getTimezoneOffset()); return d.toISOString().slice(0, 16); };
|
||||
const field = (label, el, hint) => h('label', {}, label, el, hint ? h('span', {class:'sub'}, hint) : null);
|
||||
const chk = (label, checked, onchange) => { const i = h('input', {type:'checkbox', checked: !!checked}); if (onchange) i.onchange = () => onchange(i.checked); return Object.assign(h('label', {class:'chk'}, i, label), {box:i}); };
|
||||
const select = (options, value, props) => h('select', props || {}, options.map(([v, t]) => h('option', {value:v, selected:v === value}, t)));
|
||||
const card = (title, ...kids) => h('div', {class:'card'}, title ? h('h2', {}, title) : null, ...kids);
|
||||
function dialogForm(title, intro, fields, label, submit) {
|
||||
const close = modal(title, h('div', {class:'stack'}, intro ? h('p', {class:'dim'}, intro) : null, ...fields,
|
||||
h('div', {class:'actions form-actions'}, h('button', {class:'ghost', onclick:() => close()}, 'Cancel'), h('button', {class:'btn', onclick: act(async () => { await submit(); close(); })}, label))), {sticky:true});
|
||||
}
|
||||
const cmdBox = t => h('pre', {class:'mono'}, t);
|
||||
// A dialog: focus moves in, Tab stays inside, Escape closes, and focus goes back to what opened it.
|
||||
let dialogSeq = 0;
|
||||
function modal(title, body, opts = {}) {
|
||||
const opener = document.activeElement, id = 'dialog-title-' + (++dialogSeq);
|
||||
const shade = h('div', {class:'shade'});
|
||||
const box = h('div', {class:'modal' + (opts.wide ? ' wide' : ''), role:'dialog', 'aria-modal':'true', 'aria-labelledby':id, tabindex:'-1'}, h('h2', {id}, title), body);
|
||||
let closed = false;
|
||||
const close = () => {
|
||||
if (closed) return; closed = true;
|
||||
document.removeEventListener('keydown', onKey, true); shade.remove();
|
||||
if (opener && opener.isConnected && opener.focus) opener.focus();
|
||||
if (opts.onClose) opts.onClose();
|
||||
};
|
||||
const focusables = () => [...box.querySelectorAll('a[href],button:not([disabled]),input:not([disabled]),select:not([disabled]),textarea:not([disabled]),[tabindex]:not([tabindex="-1"])')].filter(e => e.offsetParent !== null);
|
||||
function onKey(e) {
|
||||
const top = [...document.querySelectorAll('.shade')].pop();
|
||||
if (top !== shade) return;
|
||||
if (e.key === 'Escape') { e.stopPropagation(); close(); }
|
||||
else if (e.key === 'Tab') {
|
||||
const f = focusables(); if (!f.length) { e.preventDefault(); return; }
|
||||
const first = f[0], last = f[f.length - 1];
|
||||
if (e.shiftKey && (document.activeElement === first || document.activeElement === box)) { e.preventDefault(); last.focus(); }
|
||||
else if (!e.shiftKey && document.activeElement === last) { e.preventDefault(); first.focus(); }
|
||||
}
|
||||
}
|
||||
shade.addEventListener('mousedown', e => { if (e.target === shade && !opts.sticky) close(); });
|
||||
shade.append(box);
|
||||
document.body.append(shade);
|
||||
document.addEventListener('keydown', onKey, true);
|
||||
(box.querySelector('input,select,textarea') || focusables()[0] || box).focus();
|
||||
return close;
|
||||
}
|
||||
// A confirmation that names what it affects and says what is lost. Resolves true only when confirmed.
|
||||
function confirmDialog({title, body, confirm = 'Confirm', cancel = 'Cancel', danger = true}) {
|
||||
return new Promise(resolve => {
|
||||
let close;
|
||||
const done = v => { resolve(v); close(); };
|
||||
close = modal(title, h('div', {class:'stack'}, h('p', {class:'dim', style:'margin:0'}, body),
|
||||
h('div', {class:'actions form-actions'}, h('button', {class:'ghost', onclick:() => done(false)}, cancel),
|
||||
h('button', {class:danger ? 'ghost danger' : 'btn', onclick:() => done(true)}, confirm))), {onClose:() => resolve(false)});
|
||||
});
|
||||
}
|
||||
const ask = (opts, run) => act(async () => { if (await confirmDialog(opts)) await run(); });
|
||||
// status pill: color always travels with a text label
|
||||
const badge = (text, tone) => h('span', {class:'badge' + (tone ? ' badge-' + tone : '')}, text);
|
||||
const emptyState = (name, title, text, action) => h('div', {class:'empty'}, icon(name, 22), h('h3', {}, title), h('p', {}, text), action || null);
|
||||
// placeholder rows while a page loads, so the layout does not jump
|
||||
const skeleton = () => h('div', {class:'card', role:'status', 'aria-busy':'true'}, h('span', {class:'sr-only'}, 'Loading'),
|
||||
...[60, 100, 90, 100, 75].map(w => h('div', {class:'skeleton', style:`height:16px;width:${w}%;margin-bottom:14px`})));
|
||||
const modalOpen = () => !!document.querySelector('.shade');
|
||||
const table = (head, rows) => h('div', {class:'card flush'}, h('div', {class:'scroll'}, h('table', {}, h('thead', {}, h('tr', {}, head.map(c => h('th', {scope:'col'}, c)))), h('tbody', {}, rows))));
|
||||
|
||||
// A list of rows the person can add to and remove from (people in a rotation, steps in a policy).
|
||||
function rowList(addLabel, make, initial) {
|
||||
const box = h('div', {class:'stack'});
|
||||
const add = (v) => {
|
||||
const r = make(v || {});
|
||||
const row = h('div', {class:'listrow'}, r.el, h('button', {class:'ghost quiet btn-icon', 'aria-label':'Remove', type:'button', onclick:() => row.remove()}, icon('close', 16)));
|
||||
row.get = r.get; box.insertBefore(row, addBtn);
|
||||
};
|
||||
const addBtn = h('button', {class:'ghost', type:'button', onclick:() => add()}, icon('add', 16), addLabel);
|
||||
box.append(addBtn);
|
||||
(initial || []).forEach(add);
|
||||
return Object.assign(box, {values: () => [...box.querySelectorAll('.listrow')].map(r => r.get())});
|
||||
}
|
||||
|
||||
// ---------- icons (lucide-style, static strings) ----------
|
||||
const ICONS = {
|
||||
sandbar:'<path d="M2 18c3-4 6-4 10 0s7 4 10 0"/><path d="M2 12c3-4 6-4 10 0s7 4 10 0"/><path d="M2 6c3-4 6-4 10 0s7 4 10 0"/>',
|
||||
alerts:'<path d="M6 8a6 6 0 0 1 12 0c0 7 3 9 3 9H3s3-2 3-9"/><path d="M10.3 21a1.94 1.94 0 0 0 3.4 0"/>',
|
||||
incidents:'<path d="m21.73 18-8-14a2 2 0 0 0-3.48 0l-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.73-3"/><path d="M12 9v4M12 17h.01"/>',
|
||||
oncall:'<rect x="3" y="4" width="18" height="18" rx="2"/><path d="M16 2v4M8 2v4M3 10h18"/>',
|
||||
monitors:'<path d="M22 12h-4l-3 9L9 3l-3 9H2"/>',
|
||||
topics:'<path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/>',
|
||||
channels:'<path d="m22 2-7 20-4-9-9-4z"/><path d="M22 2 11 13"/>',
|
||||
status:'<circle cx="12" cy="12" r="10"/><path d="M2 12h20"/><path d="M12 2a15.3 15.3 0 0 1 4 10 15.3 15.3 0 0 1-4 10 15.3 15.3 0 0 1-4-10 15.3 15.3 0 0 1 4-10z"/>',
|
||||
users:'<path d="M20 21v-2a4 4 0 0 0-4-4H8a4 4 0 0 0-4 4v2"/><circle cx="12" cy="7" r="4"/>',
|
||||
account:'<path d="M20 21v-2a4 4 0 0 0-4-4H8a4 4 0 0 0-4 4v2"/><circle cx="12" cy="7" r="4"/>',
|
||||
out:'<path d="M9 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h4M16 17l5-5-5-5M21 12H9"/>',
|
||||
menu:'<path d="M4 6h16M4 12h16M4 18h16"/>',
|
||||
close:'<path d="M18 6 6 18M6 6l12 12"/>',
|
||||
add:'<path d="M5 12h14M12 5v14"/>',
|
||||
check:'<path d="M20 6 9 17l-5-5"/>',
|
||||
back:'<path d="m15 18-6-6 6-6"/>',
|
||||
light:'<circle cx="12" cy="12" r="4"/><path d="M12 2v2M12 20v2M4.93 4.93l1.41 1.41M17.66 17.66l1.41 1.41M2 12h2M20 12h2M6.34 17.66l-1.41 1.41M19.07 4.93l-1.41 1.41"/>',
|
||||
dark:'<path d="M12 3a6 6 0 0 0 9 9 9 9 0 1 1-9-9Z"/>',
|
||||
alert:'<path d="m21.73 18-8-14a2 2 0 0 0-3.48 0l-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.73-3"/><path d="M12 9v4M12 17h.01"/>',
|
||||
};
|
||||
function icon(name, size = 18) {
|
||||
const s = document.createElement('span');
|
||||
s.style.display = 'inline-flex'; s.setAttribute('aria-hidden', 'true');
|
||||
s.innerHTML = `<svg class="icon" xmlns="http://www.w3.org/2000/svg" width="${size}" height="${size}" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.75" stroke-linecap="round" stroke-linejoin="round" style="width:${size}px;height:${size}px">${ICONS[name] || ''}</svg>`;
|
||||
return s;
|
||||
}
|
||||
const brand = () => h('div', {class:'brand'}, icon('sandbar', 22), 'Sandbar');
|
||||
|
||||
// ---------- state ----------
|
||||
const S = { me: null, tab: 'alerts', arg: '', pub: null, users: [], policies: [], schedules: [], headerAction: null };
|
||||
const isAdmin = () => S.me && S.me.role === 'admin';
|
||||
const userName = id => (S.users.find(u => u.id === id) || {}).name || '';
|
||||
const userOpts = (blank) => [...(blank ? [['', blank]] : []), ...S.users.map(u => [u.id, u.name])];
|
||||
const origin = () => location.origin;
|
||||
|
||||
async function boot() {
|
||||
try { S.pub = await api('GET', V1 + '/state'); } catch (e) { $app.replaceChildren(card('Cannot reach the server', e.message)); return; }
|
||||
if (S.pub.needsSetup) return $app.replaceChildren(setupView());
|
||||
S.me = S.pub.user;
|
||||
if (!S.me) return $app.replaceChildren(loginView());
|
||||
await render();
|
||||
}
|
||||
|
||||
// ---------- first run and login ----------
|
||||
function setupView() {
|
||||
const f = { name: h('input', {placeholder:'Your name'}), email: h('input', {type:'email'}), pw: h('input', {type:'password'}) };
|
||||
return h('div', {class:'center'}, card('', brand(), h('h2', {}, 'Welcome to Sandbar'), h('p', {class:'dim'}, 'Create the first administrator.'),
|
||||
h('div', {class:'stack'}, field('Your name', f.name), field('Email', f.email), field('Password', f.pw, 'At least 10 characters'),
|
||||
h('button', {class:'btn', onclick: act(async () => {
|
||||
await api('POST', V1 + '/setup', {name:f.name.value, email:f.email.value, password:f.pw.value}); boot();
|
||||
})}, 'Create account'))));
|
||||
}
|
||||
function loginView() {
|
||||
const email = h('input', {type:'email', autocomplete:'username'}), pw = h('input', {type:'password', autocomplete:'current-password'});
|
||||
const go = act(async () => { await api('POST', V1 + '/login', {email:email.value, password:pw.value}); if (toastBox) toastBox.replaceChildren(); boot(); });
|
||||
[email, pw].forEach(i => i.addEventListener('keydown', e => e.key === 'Enter' && go()));
|
||||
return h('div', {class:'center'}, card('', brand(), h('h2', {}, 'Sign in'), h('div', {class:'stack'}, field('Email', email), field('Password', pw), h('button', {class:'btn', onclick:go}, 'Sign in'))));
|
||||
}
|
||||
|
||||
// ---------- shell ----------
|
||||
const PAGES = {
|
||||
alerts: ['Alerts', 'Problems that need someone, from monitors and from anything that can send a request.'],
|
||||
incidents: ['Incidents', 'Coordinate the response when something is wrong, and keep a record of what happened.'],
|
||||
oncall: ['On-call', 'Who is responsible right now, and who is told next when nobody answers.'],
|
||||
monitors: ['Monitors', 'Checks that tell you when a website, port, certificate or scheduled job stops working.'],
|
||||
topics: ['Topics', 'Publish and subscribe to messages with plain HTTP requests. Phones can use the ntfy app.'],
|
||||
channels: ['Channels', 'Email addresses and webhooks that receive every alert and incident update.'],
|
||||
users: ['People', 'Everyone who can sign in, and how to reach them.'],
|
||||
account: ['Account', 'Your password, API keys and personal topic.'],
|
||||
};
|
||||
const NAV = [
|
||||
['Respond', [['alerts', 'Alerts', 'alerts'], ['incidents', 'Incidents', 'incidents'], ['oncall', 'On-call', 'oncall']]],
|
||||
['Observe', [['monitors', 'Monitors', 'monitors'], ['topics', 'Topics', 'topics']]],
|
||||
['Set up', [['channels', 'Channels', 'channels'], ['users', 'People', 'users']]],
|
||||
];
|
||||
function route() {
|
||||
const [tab, arg] = (location.hash.replace(/^#\/?/, '') || 'alerts').split('/');
|
||||
S.tab = PAGES[tab] ? tab : 'alerts'; S.arg = arg || '';
|
||||
}
|
||||
window.addEventListener('hashchange', () => { if (S.me) { route(); render(); } });
|
||||
|
||||
async function render() {
|
||||
route();
|
||||
const nav = (key, label, ic) => h('a', {class:'navitem' + (S.tab === key ? ' on' : ''), href:'#' + key, 'aria-current':S.tab === key ? 'page' : null, onclick:() => setDrawer(false)}, icon(ic), label);
|
||||
const side = [h('div', {class:'sidebar-head'}, brand(), h('button', {class:'ghost quiet btn-icon sidebar-close', 'aria-label':'Close menu', onclick:() => setDrawer(false)}, icon('close')))];
|
||||
for (const [sec, items] of NAV) {
|
||||
side.push(h('div', {class:'navsec'}, sec));
|
||||
for (const [k, l, i] of items) side.push(nav(k, l, i));
|
||||
}
|
||||
side.push(h('div', {class:'navsec'}, 'Public'), h('a', {class:'navitem', href:'/status', target:'_blank', rel:'noopener'}, icon('status'), 'Status page'));
|
||||
side.push(h('div', {class:'grow'}), h('div', {class:'userbox'}, h('div', {class:'who'}, S.me.email), nav('account', 'Account', 'account'),
|
||||
h('button', {class:'navitem', onclick:toggleTheme}, icon('dark'), 'Switch theme'),
|
||||
h('button', {class:'navitem', onclick: act(async () => { await api('POST', V1 + '/logout'); S.me = null; boot(); })}, icon('out'), 'Sign out')));
|
||||
S.drawn = null;
|
||||
$app.replaceChildren(h('a', {class:'skip', href:'#view'}, 'Skip to content'), h('div', {class:'shell'},
|
||||
h('div', {class:'topbar'}, brand(), h('button', {class:'ghost quiet btn-icon', id:'menu-btn', 'aria-label':'Open menu', 'aria-controls':'sidebar', 'aria-expanded':'false', onclick:() => setDrawer(true)}, icon('menu'))),
|
||||
h('div', {class:'backdrop', id:'backdrop', onclick:() => setDrawer(false)}),
|
||||
h('aside', {class:'sidebar', id:'sidebar', 'aria-label':'Main'}, side),
|
||||
h('div', {class:'content'}, h('main', {id:'view', tabindex:'-1'}))));
|
||||
await refresh();
|
||||
}
|
||||
// the sidebar is a slide-in drawer on small screens
|
||||
function setDrawer(open) {
|
||||
const sb = document.getElementById('sidebar'), bd = document.getElementById('backdrop'), mb = document.getElementById('menu-btn');
|
||||
if (!sb) return;
|
||||
sb.classList.toggle('open', open); bd.classList.toggle('open', open); mb.setAttribute('aria-expanded', open);
|
||||
if (open) { const f = sb.querySelector('a,button'); if (f) f.focus(); } else mb.focus();
|
||||
}
|
||||
document.addEventListener('keydown', e => { if (e.key === 'Escape' && !modalOpen()) { const sb = document.getElementById('sidebar'); if (sb && sb.classList.contains('open')) setDrawer(false); } });
|
||||
|
||||
function pageHeader(tab) {
|
||||
const [t, d] = PAGES[tab] || ['', ''];
|
||||
const a = S.headerAction;
|
||||
return h('div', {class:'page-h'}, h('div', {}, h('h1', {}, t), d ? h('p', {}, d) : null), a ? h('button', {class:'btn', onclick:a[1]}, icon('add', 16), a[0]) : null);
|
||||
}
|
||||
let seq = 0;
|
||||
async function refresh() {
|
||||
const my = ++seq;
|
||||
try {
|
||||
S.headerAction = null;
|
||||
const first = document.getElementById('view');
|
||||
const key = S.tab + '/' + S.arg;
|
||||
if (first && S.drawn !== key) first.replaceChildren(pageHeader(S.tab), skeleton());
|
||||
[S.users, S.policies, S.schedules] = await Promise.all([api('GET', V1 + '/users'), api('GET', V1 + '/policies'), api('GET', V1 + '/schedules')]);
|
||||
const view = await views[S.tab]();
|
||||
if (my !== seq) return;
|
||||
const v = document.getElementById('view');
|
||||
if (v) { v.replaceChildren(S.noHeader ? '' : pageHeader(S.tab), view); S.drawn = key; }
|
||||
} catch (e) { if (e.message !== SIGNED_OUT) toast(e.message, true); }
|
||||
}
|
||||
setInterval(() => {
|
||||
if (!S.me || document.hidden || modalOpen()) return;
|
||||
const a = document.activeElement;
|
||||
if (a && ['INPUT', 'SELECT', 'TEXTAREA'].includes(a.tagName)) return;
|
||||
if (['alerts', 'monitors', 'incidents', 'oncall'].includes(S.tab) && !S.arg) refresh();
|
||||
}, 10000);
|
||||
|
||||
const sevTone = s => ({critical:'danger', warning:'warning'}[s] || '');
|
||||
const sevLabel = s => ({critical:'Critical', warning:'Warning', info:'Info'}[s] || s);
|
||||
const alertStatus = a => a.status === 'firing' ? badge('Firing', 'danger') : a.status === 'acked' ? badge('Acknowledged', 'warning') : badge('Resolved', 'success');
|
||||
const timeline = ev => h('ul', {class:'timeline'}, ev.map(e => h('li', {}, h('span', {class:'when num'}, fmtTime(e.time)), h('span', {}, (e.actor ? e.actor + ': ' : '') + e.text))));
|
||||
|
||||
const views = {};
|
||||
|
||||
// ---------- alerts ----------
|
||||
views.alerts = async () => {
|
||||
const filter = S.alertFilter || 'open';
|
||||
S.headerAction = ['Raise alert', () => raiseAlertDialog()];
|
||||
const list = await api('GET', V1 + '/alerts?status=' + filter);
|
||||
const tabs = h('div', {class:'tabs2', role:'group', 'aria-label':'Filter alerts'}, [['open', 'Open'], ['resolved', 'Resolved'], ['all', 'All']].map(([k, l]) =>
|
||||
h('button', {class:filter === k ? 'on' : '', 'aria-pressed':filter === k, onclick:() => { S.alertFilter = k; refresh(); }}, l)));
|
||||
if (!list.length) return h('div', {}, tabs, emptyState('alerts', filter === 'open' ? 'No open alerts' : 'No alerts yet', 'Alerts appear here when a monitor goes down or something sends one to the API.'));
|
||||
return h('div', {}, tabs, table(['Severity', 'Alert', 'Status', 'Started', ''], list.map(a => h('tr', {},
|
||||
h('td', {}, badge(sevLabel(a.severity), sevTone(a.severity))),
|
||||
h('td', {}, h('div', {}, a.summary), a.detail ? h('div', {class:'small dim clip', title:a.detail}, a.detail) : null, a.count > 1 ? h('div', {class:'small dim'}, 'Seen ' + a.count + ' times') : null),
|
||||
h('td', {}, alertStatus(a), a.status === 'acked' ? h('div', {class:'small dim'}, 'by ' + a.ackedBy) : null),
|
||||
h('td', {class:'num'}, fmtTime(a.created)),
|
||||
h('td', {}, h('div', {class:'actions'},
|
||||
a.status === 'firing' ? h('button', {class:'ghost', onclick:act(async () => { await api('POST', `${V1}/alerts/${a.id}/ack`); toast('Alert acknowledged.'); refresh(); })}, 'Acknowledge') : null,
|
||||
a.status !== 'resolved' ? h('button', {class:'ghost', onclick:act(async () => { await api('POST', `${V1}/alerts/${a.id}/resolve`); toast('Alert resolved.'); refresh(); })}, 'Resolve') : null,
|
||||
h('button', {class:'ghost', onclick:act(() => alertDetail(a.id))}, 'Details')))))));
|
||||
};
|
||||
async function alertDetail(id) {
|
||||
const a = await api('GET', `${V1}/alerts/${id}`);
|
||||
const pol = S.policies.find(p => p.id === a.policyId);
|
||||
const body = h('div', {class:'stack'}, h('div', {}, alertStatus(a), ' ', badge(sevLabel(a.severity), sevTone(a.severity))),
|
||||
a.detail ? h('p', {class:'dim', style:'margin:0'}, a.detail) : null,
|
||||
h('p', {class:'small dim', style:'margin:0'}, 'Escalation policy: ' + (pol ? pol.name : 'none, broadcast only')),
|
||||
timeline(a.timeline || []),
|
||||
h('div', {class:'actions form-actions'},
|
||||
a.status === 'firing' ? h('button', {class:'ghost', onclick:act(async () => { await api('POST', `${V1}/alerts/${id}/snooze`, {minutes:30}); toast('Escalation paused for 30 minutes.'); close(); refresh(); })}, 'Snooze 30 minutes') : null,
|
||||
a.status !== 'resolved' && !a.incidentId ? h('button', {class:'ghost', onclick:() => { close(); declareDialog([a.id], a.summary); }}, 'Declare incident') : null,
|
||||
h('button', {class:'btn', onclick:() => close()}, 'Close')));
|
||||
const close = modal(a.summary, body, {wide:true});
|
||||
}
|
||||
function raiseAlertDialog() {
|
||||
const f = { summary: h('input'), detail: h('textarea'), sev: select([['critical', 'Critical'], ['warning', 'Warning'], ['info', 'Info']], 'warning'),
|
||||
pol: select([['', 'None, only broadcast'], ...S.policies.map(p => [p.id, p.name])], '') };
|
||||
dialogForm('Raise an alert', 'Use this to test an escalation policy, or to page someone by hand.',
|
||||
[field('Summary', f.summary), field('Details', f.detail), field('Severity', f.sev), field('Escalation policy', f.pol)], 'Raise alert',
|
||||
async () => { await api('POST', V1 + '/alerts', {key:'manual-' + Date.now(), summary:f.summary.value, detail:f.detail.value, severity:f.sev.value, policy:f.pol.value, source:'manual'}); toast('Alert raised.'); refresh(); });
|
||||
}
|
||||
|
||||
// ---------- monitors ----------
|
||||
const KINDS = [['http', 'Website (HTTP)'], ['tcp', 'Port (TCP)'], ['dns', 'DNS lookup'], ['tls', 'Certificate expiry'], ['heartbeat', 'Heartbeat (cron job or service pings us)']];
|
||||
views.monitors = async () => {
|
||||
S.headerAction = ['Add monitor', () => monitorDialog()];
|
||||
const [mons, states] = await Promise.all([api('GET', V1 + '/monitors'), api('GET', V1 + '/monitor-states')]);
|
||||
if (!mons.length) return emptyState('monitors', 'No monitors yet', 'Add a check for a website, a port, a certificate or a scheduled job. Failures raise an alert.', h('button', {class:'btn', onclick:() => monitorDialog()}, 'Add monitor'));
|
||||
const statusBadge = (m, st) => !m.enabled ? badge('Paused') : st.status === 'up' ? badge('Up', 'success') : st.status === 'down' ? badge('Down', 'danger') : badge('Waiting for first check');
|
||||
return table(['Name', 'Check', 'Status', 'Last check', 'Uptime', ''], mons.map(m => { const st = states[m.id] || {};
|
||||
return h('tr', {},
|
||||
h('td', {}, m.name, m.public ? h('div', {}, badge('On status page', 'accent')) : null),
|
||||
h('td', {}, h('div', {}, (KINDS.find(k => k[0] === m.kind) || [])[1] || m.kind), m.kind !== 'heartbeat' ? h('div', {class:'small dim mono'}, m.target) : h('div', {class:'small dim'}, 'Every ' + m.intervalS + ' s')),
|
||||
h('td', {}, statusBadge(m, st), st.status === 'down' && st.detail ? h('div', {class:'small dim clip', title:st.detail}, st.detail) : null),
|
||||
h('td', {class:'num'}, st.lastCheck ? ago(st.lastCheck) : 'never', m.kind !== 'heartbeat' && st.lastCheck ? h('div', {class:'small dim'}, st.lastMs + ' ms') : null),
|
||||
h('td', {class:'num'}, st.uptime30 >= 0 ? st.uptime30.toFixed(2) + '%' : 'No data yet'),
|
||||
h('td', {}, h('div', {class:'actions'},
|
||||
h('button', {class:'ghost', onclick:() => monitorDialog(m)}, 'Edit'),
|
||||
h('button', {class:'ghost danger', onclick:ask({title:`Delete ${m.name}?`, body:'The monitor, its history and any open alert for it are removed. This cannot be undone.', confirm:'Delete monitor'}, async () => { await api('DELETE', `${V1}/monitors/${m.id}`); toast('Monitor deleted.'); refresh(); })}, 'Delete'))));
|
||||
}));
|
||||
};
|
||||
function monitorDialog(m) {
|
||||
const e = !!m; m = m || {kind:'http', intervalS:60, timeoutS:10, retries:2, severity:'critical', enabled:true, warnDays:14};
|
||||
const f = { name: h('input', {value:m.name || ''}), kind: select(KINDS, m.kind, {disabled:e}), target: h('input', {value:m.target || ''}),
|
||||
interval: h('input', {type:'number', min:10, value:m.intervalS}), timeout: h('input', {type:'number', min:1, max:120, value:m.timeoutS}),
|
||||
retries: h('input', {type:'number', min:1, value:m.retries}), expect: h('input', {value:m.expect || '', placeholder:'200-299'}), keyword: h('input', {value:m.keyword || ''}),
|
||||
warn: h('input', {type:'number', min:1, value:m.warnDays || 14}), grace: h('input', {type:'number', min:0, value:m.graceS || 0}),
|
||||
pol: select([['', 'None, only broadcast'], ...S.policies.map(p => [p.id, p.name])], m.policyId || ''),
|
||||
sev: select([['critical', 'Critical'], ['warning', 'Warning'], ['info', 'Info']], m.severity || 'critical'),
|
||||
pub: chk('Show on the public status page', m.public), priv: chk('Allow targets on private networks', m.allowPrivate), on: chk('Run this check', m.enabled) };
|
||||
const only = (kinds, el) => { const w = h('div', {class:'stack'}, el); w.kinds = kinds; return w; };
|
||||
const groups = [
|
||||
only(['http', 'tcp', 'dns', 'tls'], field('Target', f.target, 'A URL for websites, host:port for ports, a hostname for DNS and certificates.')),
|
||||
only(['http'], field('Expected status', f.expect, 'Leave empty for any 2xx. A range such as 200-399 also works.')),
|
||||
only(['http'], field('Page must contain', f.keyword, 'Optional text that has to appear in the response.')),
|
||||
only(['tls'], field('Alert when expiring within (days)', f.warn)),
|
||||
only(['heartbeat'], field('Allowed delay after the interval (seconds)', f.grace)),
|
||||
only(['http', 'tcp', 'dns', 'tls'], f.priv),
|
||||
];
|
||||
const sync = () => groups.forEach(g => g.style.display = g.kinds.includes(f.kind.value) ? '' : 'none');
|
||||
f.kind.onchange = sync; sync();
|
||||
const pingUrl = m.token ? h('div', {}, h('div', {class:'small dim'}, 'Send a request here every time the job runs:'), cmdBox(`${origin()}/api/v1/hb/${m.token}`)) : null;
|
||||
const hbOnly = only(['heartbeat'], pingUrl || h('p', {class:'small dim', style:'margin:0'}, 'The ping address appears here after you save.'));
|
||||
groups.push(hbOnly); sync();
|
||||
dialogForm(e ? 'Edit ' + m.name : 'Add a monitor', null, [field('Name', f.name), field('Type', f.kind), ...groups,
|
||||
h('div', {class:'grid'}, field('Check every (seconds)', f.interval, 'At least 10.'), field('Timeout (seconds)', f.timeout), field('Failures before down', f.retries, 'A blip is ignored.')),
|
||||
h('div', {class:'grid'}, field('Alert severity', f.sev), field('Escalation policy', f.pol)), f.pub, f.on], e ? 'Save changes' : 'Add monitor',
|
||||
async () => {
|
||||
const body = {...m, name:f.name.value, kind:f.kind.value, target:f.target.value, intervalS:+f.interval.value, timeoutS:+f.timeout.value, retries:+f.retries.value,
|
||||
expect:f.expect.value, keyword:f.keyword.value, warnDays:+f.warn.value, graceS:+f.grace.value, policyId:f.pol.value, severity:f.sev.value,
|
||||
public:f.pub.box.checked, allowPrivate:f.priv.box.checked, enabled:f.on.box.checked};
|
||||
const saved = await api(e ? 'PUT' : 'POST', V1 + '/monitors' + (e ? '/' + m.id : ''), body);
|
||||
toast(e ? 'Monitor saved.' : 'Monitor added.'); refresh();
|
||||
if (!e && saved.kind === 'heartbeat') { const done = modal('Heartbeat address', h('div', {class:'stack'}, h('p', {class:'dim', style:'margin:0'}, 'Have your job request this address each time it runs. It goes down when the requests stop.'), cmdBox(`${origin()}/api/v1/hb/${saved.token}`), h('div', {class:'actions form-actions'}, h('button', {class:'btn', onclick:() => done()}, 'Done')))); }
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- on-call ----------
|
||||
views.oncall = async () => {
|
||||
const which = S.oncallTab || 'schedules';
|
||||
S.headerAction = which === 'schedules' ? ['Add schedule', () => scheduleDialog()] : ['Add policy', () => policyDialog()];
|
||||
const tabs = h('div', {class:'tabs2', role:'group', 'aria-label':'On-call sections'}, [['schedules', 'Schedules'], ['policies', 'Escalation policies']].map(([k, l]) =>
|
||||
h('button', {class:which === k ? 'on' : '', 'aria-pressed':which === k, onclick:() => { S.oncallTab = k; refresh(); }}, l)));
|
||||
if (which === 'schedules') {
|
||||
const now = await api('GET', V1 + '/oncall');
|
||||
if (!S.schedules.length) return h('div', {}, tabs, emptyState('oncall', 'No schedules yet', 'A schedule rotates a list of people. Use it in an escalation policy so the right person is told.', h('button', {class:'btn', onclick:() => scheduleDialog()}, 'Add schedule')));
|
||||
return h('div', {}, tabs, table(['Schedule', 'On call now', 'Next', 'Rotation', ''], S.schedules.map(sc => { const n = now.find(x => x.id === sc.id) || {};
|
||||
return h('tr', {}, h('td', {}, sc.name), h('td', {}, n.user ? badge(n.user, 'accent') : badge('Nobody')), h('td', {}, n.next || 'Nobody'),
|
||||
h('td', {}, `${sc.members.length} ${sc.members.length === 1 ? 'person' : 'people'}, every ${sc.rotationHours % 24 === 0 ? sc.rotationHours / 24 + (sc.rotationHours === 24 ? ' day' : ' days') : sc.rotationHours + ' hours'}`),
|
||||
h('td', {}, h('div', {class:'actions'}, h('button', {class:'ghost', onclick:() => scheduleDialog(sc)}, 'Edit'),
|
||||
h('button', {class:'ghost danger', onclick:ask({title:`Delete ${sc.name}?`, body:'Policies that use this schedule will have nobody to notify for that step. This cannot be undone.', confirm:'Delete schedule'}, async () => { await api('DELETE', `${V1}/schedules/${sc.id}`); toast('Schedule deleted.'); refresh(); })}, 'Delete'))));
|
||||
})));
|
||||
}
|
||||
if (!S.policies.length) return h('div', {}, tabs, emptyState('oncall', 'No escalation policies yet', 'A policy lists who to tell, and how long to wait before telling the next person.', h('button', {class:'btn', onclick:() => policyDialog()}, 'Add policy')));
|
||||
const who = st => [...st.users.map(userName), ...st.schedules.map(id => 'on call for ' + ((S.schedules.find(s => s.id === id) || {}).name || 'a removed schedule'))].filter(Boolean).join(', ') || 'Nobody';
|
||||
return h('div', {}, tabs, table(['Policy', 'Steps', ''], S.policies.map(p => h('tr', {}, h('td', {}, p.name),
|
||||
h('td', {}, h('ol', {class:'steps'}, p.steps.map(st => h('li', {}, (st.delayMin ? `After ${st.delayMin} min: ` : 'At once: ') + who(st)))), p.repeatMin ? h('div', {class:'small dim'}, `Starts over every ${p.repeatMin} min until acknowledged.`) : null),
|
||||
h('td', {}, h('div', {class:'actions'}, h('button', {class:'ghost', onclick:() => policyDialog(p)}, 'Edit'),
|
||||
h('button', {class:'ghost danger', onclick:ask({title:`Delete ${p.name}?`, body:'Monitors that use this policy will only broadcast. Alerts already escalating stop. This cannot be undone.', confirm:'Delete policy'}, async () => { await api('DELETE', `${V1}/policies/${p.id}`); toast('Policy deleted.'); refresh(); })}, 'Delete')))))));
|
||||
};
|
||||
function scheduleDialog(sc) {
|
||||
const e = !!sc; sc = sc || {members:[], overrides:[], rotationHours:168};
|
||||
const name = h('input', {value:sc.name || ''}), start = h('input', {type:'datetime-local', value:fromUnix(sc.start || Date.now() / 1000)});
|
||||
const preset = [[24, 'Every day'], [168, 'Every week'], [336, 'Every two weeks']];
|
||||
const rot = select([...preset, ['custom', 'Custom']], preset.some(p => p[0] === sc.rotationHours) ? sc.rotationHours : 'custom'), hours = h('input', {type:'number', min:1, value:sc.rotationHours});
|
||||
const sync = () => { hours.parentElement.style.display = rot.value === 'custom' ? '' : 'none'; if (rot.value !== 'custom') hours.value = rot.value; };
|
||||
rot.onchange = sync;
|
||||
const members = rowList('Add person', v => { const s = select(userOpts(), v.id || S.users[0]?.id); return {el:field('Person', s), get:() => s.value}; }, sc.members.map(id => ({id})));
|
||||
const overrides = rowList('Add override', v => { const u = select(userOpts(), v.userId), a = h('input', {type:'datetime-local', value:fromUnix(v.from)}), b = h('input', {type:'datetime-local', value:fromUnix(v.to)});
|
||||
return {el:h('div', {class:'grid'}, field('Covered by', u), field('From', a), field('Until', b)), get:() => ({userId:u.value, from:toUnix(a.value), to:toUnix(b.value)})}; }, sc.overrides);
|
||||
const hw = field('Hours per shift', hours);
|
||||
dialogForm(e ? 'Edit ' + sc.name : 'Add a schedule', 'People take turns in the order listed.', [field('Name', name), field('Hand over', rot), hw,
|
||||
field('First shift starts', start), h('fieldset', {}, h('legend', {}, 'People in the rotation'), members), h('fieldset', {}, h('legend', {}, 'Overrides'), h('p', {class:'sub'}, 'Cover a holiday or swap a shift for a set time.'), overrides)],
|
||||
e ? 'Save changes' : 'Add schedule', async () => {
|
||||
await api(e ? 'PUT' : 'POST', V1 + '/schedules' + (e ? '/' + sc.id : ''), {name:name.value, rotationHours:+hours.value, start:toUnix(start.value), members:members.values(), overrides:overrides.values()});
|
||||
toast('Schedule saved.'); refresh(); });
|
||||
sync();
|
||||
}
|
||||
function policyDialog(p) {
|
||||
const e = !!p; p = p || {steps:[{delayMin:0, users:[], schedules:[]}], repeatMin:0};
|
||||
const name = h('input', {value:p.name || ''}), repeat = h('input', {type:'number', min:0, value:p.repeatMin || 0});
|
||||
const steps = rowList('Add step', v => {
|
||||
const delay = h('input', {type:'number', min:0, value:v.delayMin || 0});
|
||||
const users = S.users.map(u => chk(u.name, (v.users || []).includes(u.id))), scs = S.schedules.map(s => chk(s.name, (v.schedules || []).includes(s.id)));
|
||||
return {el:h('div', {class:'stack'}, field('Wait before this step (minutes)', delay), h('div', {class:'pillrow'}, users, scs.length ? h('span', {class:'sub'}, 'On call for:') : null, scs)),
|
||||
get:() => ({delayMin:+delay.value, users:S.users.filter((u, i) => users[i].box.checked).map(u => u.id), schedules:S.schedules.filter((s, i) => scs[i].box.checked).map(s => s.id)})};
|
||||
}, p.steps);
|
||||
dialogForm(e ? 'Edit ' + p.name : 'Add an escalation policy', 'Each step tells these people. If nobody acknowledges, the next step runs after its wait.', [field('Name', name), h('fieldset', {}, h('legend', {}, 'Steps'), steps),
|
||||
field('Start over after (minutes)', repeat, 'Zero stops after the last step.')], e ? 'Save changes' : 'Add policy', async () => {
|
||||
await api(e ? 'PUT' : 'POST', V1 + '/policies' + (e ? '/' + p.id : ''), {name:name.value, steps:steps.values(), repeatMin:+repeat.value}); toast('Policy saved.'); refresh(); });
|
||||
}
|
||||
|
||||
// ---------- incidents ----------
|
||||
const incTone = s => ({sev1:'danger', sev2:'warning'}[s] || '');
|
||||
const incStatus = s => s === 'resolved' ? badge('Resolved', 'success') : s === 'monitoring' ? badge('Monitoring', 'accent') : s === 'identified' ? badge('Identified', 'warning') : badge('Investigating', 'danger');
|
||||
const STATUSES = [['investigating', 'Investigating'], ['identified', 'Identified'], ['monitoring', 'Monitoring'], ['resolved', 'Resolved']];
|
||||
const SEVS = [['sev1', 'Severity 1, major outage'], ['sev2', 'Severity 2, partial impact'], ['sev3', 'Severity 3, minor']];
|
||||
views.incidents = async () => {
|
||||
if (S.arg) return incidentDetail(S.arg);
|
||||
S.headerAction = ['Declare incident', () => declareDialog()];
|
||||
const list = await api('GET', V1 + '/incidents');
|
||||
if (!list.length) return emptyState('incidents', 'No incidents yet', 'Declare an incident when something is wrong and people need to work together. Every step is kept in a timeline.', h('button', {class:'btn', onclick:() => declareDialog()}, 'Declare incident'));
|
||||
return table(['Severity', 'Incident', 'Status', 'Lead', 'Started'], list.map(i => h('tr', {},
|
||||
h('td', {}, badge(i.severity.replace('sev', 'Sev '), incTone(i.severity))),
|
||||
h('td', {}, h('a', {href:'#incidents/' + i.id}, i.title), i.public ? h('div', {}, badge('On status page', 'accent')) : null),
|
||||
h('td', {}, incStatus(i.status)), h('td', {}, userName(i.lead) || 'Nobody'), h('td', {class:'num'}, fmtTime(i.created)))));
|
||||
};
|
||||
function declareDialog(alertIds, title) {
|
||||
const f = { title: h('input', {value:title || ''}), sev: select(SEVS, 'sev2'), sum: h('textarea'), lead: select(userOpts('Me'), '') };
|
||||
dialogForm('Declare an incident', 'Everyone subscribed to the incidents topic and every channel is told.', [field('Title', f.title, 'What is wrong, in a few words.'), field('Severity', f.sev), field('What we know so far', f.sum), field('Incident lead', f.lead)], 'Declare incident',
|
||||
async () => { const i = await api('POST', V1 + '/incidents', {title:f.title.value, severity:f.sev.value, summary:f.sum.value, lead:f.lead.value, alertIds:alertIds || []}); toast('Incident declared.'); location.hash = 'incidents/' + i.id; });
|
||||
}
|
||||
async function incidentDetail(id) {
|
||||
S.noHeader = true;
|
||||
const i = await api('GET', `${V1}/incidents/${id}`);
|
||||
const patch = act(async (body, msg) => { await api('PATCH', `${V1}/incidents/${id}`, body); toast(msg || 'Incident updated.'); refresh(); });
|
||||
const status = select(STATUSES, i.status, {onchange:e => patch({status:e.target.value})}), sev = select(SEVS, i.severity, {onchange:e => patch({severity:e.target.value})});
|
||||
const lead = select(userOpts('Nobody'), i.lead, {onchange:e => patch({lead:e.target.value})}), comms = select(userOpts('Nobody'), i.comms, {onchange:e => patch({comms:e.target.value})});
|
||||
const pub = chk('Show on the public status page', i.public, v => patch({public:v}));
|
||||
const sum = h('textarea', {}, i.summary), pm = h('textarea', {style:'min-height:220px', class:'mono'}, i.postmortem), note = h('input', {placeholder:'Add a comment for the timeline'});
|
||||
const say = act(async () => { if (!note.value.trim()) return; await api('POST', `${V1}/incidents/${id}/comments`, {text:note.value}); note.value = ''; refresh(); });
|
||||
note.addEventListener('keydown', e => e.key === 'Enter' && say());
|
||||
setTimeout(() => { S.noHeader = false; });
|
||||
return h('div', {}, h('div', {class:'page-h'}, h('div', {}, h('a', {href:'#incidents', class:'small'}, 'Back to incidents'), h('h1', {}, i.title), h('p', {}, 'Declared ' + fmtTime(i.created) + (i.resolved ? ', resolved ' + fmtTime(i.resolved) : ''))), h('div', {}, incStatus(i.status))),
|
||||
card('Status', h('div', {class:'grid'}, field('Status', status), field('Severity', sev), field('Lead', lead), field('Communications', comms)), pub),
|
||||
card('Summary', h('div', {class:'stack'}, sum, h('div', {class:'actions form-actions'}, h('button', {class:'ghost', onclick:() => patch({summary:sum.value}, 'Summary saved.')}, 'Save summary')))),
|
||||
card('Timeline', timeline(i.timeline || []), h('div', {class:'actions', style:'margin-top:16px'}, h('div', {style:'flex:1;min-width:200px'}, note), h('button', {class:'ghost', onclick:say}, 'Add comment'))),
|
||||
card('Post-mortem', h('div', {class:'stack'}, pm, h('div', {class:'actions form-actions'},
|
||||
h('button', {class:'ghost', onclick:act(async () => { if (pm.value.trim() && !await confirmDialog({title:'Replace the post-mortem?', body:'The text you have written is replaced by the template. This cannot be undone.', confirm:'Use template'})) return; pm.value = (await api('POST', `${V1}/incidents/${id}/postmortem-template`)).template; })}, 'Fill from template'),
|
||||
h('button', {class:'btn', onclick:() => patch({postmortem:pm.value}, 'Post-mortem saved.')}, 'Save post-mortem')))),
|
||||
isAdmin() ? h('div', {class:'actions'}, h('button', {class:'ghost danger', onclick:ask({title:`Delete ${i.title}?`, body:'The incident and its timeline are removed. Linked alerts stay. This cannot be undone.', confirm:'Delete incident'}, async () => { await api('DELETE', `${V1}/incidents/${id}`); toast('Incident deleted.'); location.hash = 'incidents'; })}, 'Delete incident')) : null);
|
||||
}
|
||||
|
||||
// ---------- topics ----------
|
||||
let stream = null;
|
||||
views.topics = async () => {
|
||||
if (stream) { stream.close(); stream = null; }
|
||||
const topic = h('input', {value:S.topic || 'alerts', 'aria-label':'Topic'}), feed = h('div', {class:'feed', role:'log', 'aria-live':'polite'});
|
||||
const title = h('input'), msg = h('textarea'), prio = select([['3', 'Default'], ['5', 'Urgent'], ['4', 'High'], ['2', 'Low'], ['1', 'Minimum']], '3'), tags = h('input', {placeholder:'warning, disk'});
|
||||
const show = m => { feed.prepend(h('div', {class:'msg'}, h('div', {class:'small dim num'}, fmtTime(m.time) + (m.priority >= 4 ? ' · priority ' + m.priority : '')), m.title ? h('strong', {}, m.title) : null, h('div', {}, m.message), (m.tags || []).length ? h('div', {}, m.tags.map(t => h('span', {class:'tag'}, t))) : null)); };
|
||||
const empty = emptyState('topics', 'No messages yet', 'Messages published to this topic appear here as they arrive.');
|
||||
const listen = () => {
|
||||
if (stream) stream.close(); S.topic = topic.value.trim(); feed.replaceChildren(empty);
|
||||
stream = new EventSource('/' + encodeURIComponent(S.topic) + '/sse?since=all');
|
||||
stream.onmessage = ev => { if (empty.isConnected) feed.replaceChildren(); show(JSON.parse(ev.data)); };
|
||||
};
|
||||
topic.addEventListener('keydown', e => e.key === 'Enter' && listen());
|
||||
const base = origin(), mine = S.me.topic;
|
||||
setTimeout(listen);
|
||||
return h('div', {}, card('Watch a topic', h('div', {class:'stack'}, h('div', {class:'actions'}, h('div', {style:'flex:1;min-width:200px'}, field('Topic', topic)), h('button', {class:'ghost', style:'align-self:flex-end', onclick:listen}, 'Watch')),
|
||||
h('p', {class:'small dim', style:'margin:0'}, `Every alert is published to "alerts" and every incident update to "incidents". Your own topic is "${mine}": add it in the ntfy app to get paged on your phone.`), feed)),
|
||||
card('Publish a message', h('div', {class:'stack'}, field('Title', title), field('Message', msg), h('div', {class:'grid'}, field('Priority', prio), field('Tags', tags, 'Separated by commas.')),
|
||||
h('div', {class:'actions form-actions'}, h('button', {class:'btn', onclick:act(async () => {
|
||||
const r = await fetch('/' + encodeURIComponent(topic.value.trim()), {method:'POST', body:msg.value, headers:{Title:title.value, Priority:prio.value, Tags:tags.value}});
|
||||
if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || 'The message was not sent.');
|
||||
msg.value = ''; toast('Message published.'); })}, 'Publish message')))),
|
||||
card('From the command line', cmdBox(`curl -H "Authorization: Bearer <api key>" -H "Title: Disk full" -H "Priority: urgent" -d "web-1 is at 98%" ${base}/alerts\ncurl -H "Authorization: Bearer <api key>" -N ${base}/alerts/sse`)));
|
||||
};
|
||||
|
||||
// ---------- channels ----------
|
||||
views.channels = async () => {
|
||||
S.headerAction = isAdmin() ? ['Add channel', () => channelDialog()] : null;
|
||||
const list = await api('GET', V1 + '/channels');
|
||||
if (!list.length) return emptyState('channels', 'No channels yet', 'A channel sends every alert and incident update to an email address or a chat webhook such as Slack, Discord or ntfy.', isAdmin() ? h('button', {class:'btn', onclick:() => channelDialog()}, 'Add channel') : null);
|
||||
return table(['Name', 'Type', 'Status', ''], list.map(c => h('tr', {}, h('td', {}, c.name), h('td', {}, c.kind === 'email' ? 'Email' : 'Webhook (' + c.format + ')'), h('td', {}, c.enabled ? badge('Active', 'success') : badge('Paused')),
|
||||
h('td', {}, isAdmin() ? h('div', {class:'actions'}, h('button', {class:'ghost', onclick:act(async () => { await api('POST', `${V1}/channels/${c.id}/test`); toast('Test notification sent.'); })}, 'Send test'),
|
||||
h('button', {class:'ghost', onclick:() => channelDialog(c)}, 'Edit'),
|
||||
h('button', {class:'ghost danger', onclick:ask({title:`Delete ${c.name}?`, body:'It stops receiving alerts and incident updates. This cannot be undone.', confirm:'Delete channel'}, async () => { await api('DELETE', `${V1}/channels/${c.id}`); toast('Channel deleted.'); refresh(); })}, 'Delete')) : null))));
|
||||
};
|
||||
function channelDialog(c) {
|
||||
const e = !!c; c = c || {kind:'webhook', format:'slack', enabled:true};
|
||||
const f = { name: h('input', {value:c.name || ''}), kind: select([['webhook', 'Webhook'], ['email', 'Email']], c.kind), target: h('input', {value:c.target || ''}),
|
||||
format: select([['slack', 'Slack'], ['discord', 'Discord'], ['ntfy', 'ntfy'], ['json', 'Plain JSON']], c.format || 'slack'), priv: chk('Allow addresses on private networks', c.allowPrivate), on: chk('Send notifications', c.enabled) };
|
||||
const fmt = field('Webhook format', f.format), tgt = field('Address', f.target, 'Webhook URL, or email addresses separated by commas. Stored encrypted and hidden after saving.');
|
||||
const sync = () => { fmt.style.display = f.priv.style.display = f.kind.value === 'webhook' ? '' : 'none'; };
|
||||
f.kind.onchange = sync; sync();
|
||||
dialogForm(e ? 'Edit ' + c.name : 'Add a channel', null, [field('Name', f.name), field('Type', f.kind), tgt, fmt, f.priv, f.on], e ? 'Save changes' : 'Add channel', async () => {
|
||||
await api(e ? 'PUT' : 'POST', V1 + '/channels' + (e ? '/' + c.id : ''), {name:f.name.value, kind:f.kind.value, target:f.target.value, format:f.format.value, allowPrivate:f.priv.box.checked, enabled:f.on.box.checked});
|
||||
toast('Channel saved.'); refresh(); });
|
||||
}
|
||||
|
||||
// ---------- people ----------
|
||||
views.users = async () => {
|
||||
S.headerAction = isAdmin() ? ['Add person', () => userDialog()] : null;
|
||||
return table(['Name', 'Email', 'Role', 'Personal topic', ''], S.users.map(u => h('tr', {}, h('td', {}, u.name), h('td', {}, u.email), h('td', {}, u.role === 'admin' ? badge('Administrator', 'accent') : badge('Member')), h('td', {class:'mono small'}, u.topic),
|
||||
h('td', {}, isAdmin() ? h('div', {class:'actions'}, h('button', {class:'ghost', onclick:() => passwordDialog(u)}, 'Set password'),
|
||||
u.id !== S.me.id ? h('button', {class:'ghost danger', onclick:ask({title:`Delete ${u.name}?`, body:'They can no longer sign in and their API keys stop working. Schedules and policies that list them need updating. This cannot be undone.', confirm:'Delete person'}, async () => { await api('DELETE', `${V1}/users/${u.id}`); toast('Person deleted.'); refresh(); })}, 'Delete') : null) : null))));
|
||||
};
|
||||
function userDialog() {
|
||||
const f = { name: h('input'), email: h('input', {type:'email'}), pw: h('input', {type:'password'}), role: select([['member', 'Member'], ['admin', 'Administrator']], 'member'), phone: h('input', {type:'tel'}) };
|
||||
dialogForm('Add a person', null, [field('Name', f.name), field('Email', f.email), field('Phone', f.phone, 'Optional.'), field('Password', f.pw, 'At least 10 characters'), field('Role', f.role, 'Administrators manage monitors, channels, schedules, policies and people.')], 'Add person',
|
||||
async () => { await api('POST', V1 + '/users', {name:f.name.value, email:f.email.value, phone:f.phone.value, password:f.pw.value, role:f.role.value}); toast('Person added.'); refresh(); });
|
||||
}
|
||||
function passwordDialog(u) {
|
||||
const pw = h('input', {type:'password'});
|
||||
dialogForm('Set a password for ' + u.name, 'They are signed out everywhere.', [field('New password', pw, 'At least 10 characters')], 'Set password', async () => { await api('POST', `${V1}/users/${u.id}/password`, {password:pw.value}); toast('Password changed.'); });
|
||||
}
|
||||
|
||||
// ---------- account ----------
|
||||
views.account = async () => {
|
||||
const keys = await api('GET', V1 + '/apikeys'), pw = h('input', {type:'password', autocomplete:'new-password'});
|
||||
S.headerAction = null;
|
||||
return h('div', {}, card('Password', h('div', {class:'stack'}, field('New password', pw, 'At least 10 characters. You are signed out on other devices.'),
|
||||
h('div', {class:'actions form-actions'}, h('button', {class:'btn', onclick:act(async () => { await api('POST', `${V1}/users/${S.me.id}/password`, {password:pw.value}); pw.value = ''; toast('Password changed. Sign in again.'); S.me = null; boot(); })}, 'Change password')))),
|
||||
card('API keys', h('p', {class:'dim'}, 'Keys let scripts, monitors and ntfy clients publish messages and raise alerts. A key has your access.'),
|
||||
keys.length ? h('div', {class:'card flush', style:'margin-bottom:16px'}, h('table', {}, h('tbody', {}, keys.map(k => h('tr', {}, h('td', {}, k.name || 'Unnamed key'), h('td', {class:'num'}, 'Created ' + fmtDate(k.created)),
|
||||
h('td', {}, h('button', {class:'ghost danger', onclick:ask({title:`Delete ${k.name || 'this key'}?`, body:'Anything using it stops working. This cannot be undone.', confirm:'Delete key'}, async () => { await api('DELETE', `${V1}/apikeys/${k.id}`); refresh(); })}, 'Delete')))))) ) : null,
|
||||
h('button', {class:'ghost', onclick:() => { const n = h('input'); dialogForm('Create an API key', null, [field('Name', n, 'Where you will use it, such as "prometheus".')], 'Create key', async () => {
|
||||
const k = await api('POST', V1 + '/apikeys', {name:n.value}); refresh();
|
||||
const done = modal('Copy your API key', h('div', {class:'stack'}, h('p', {class:'dim', style:'margin:0'}, 'This is the only time it is shown.'), cmdBox(k.token), h('div', {class:'actions form-actions'}, h('button', {class:'btn', onclick:() => done()}, 'Done')))); }); }}, icon('add', 16), 'Create API key')),
|
||||
card('Your personal topic', h('p', {class:'dim'}, 'Subscribe to this topic in the ntfy app, or in a browser on the Topics page, to be paged when an escalation policy reaches you.'), cmdBox(S.me.topic)));
|
||||
};
|
||||
|
||||
boot();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,104 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="color-scheme" content="light dark">
|
||||
<title>Status</title>
|
||||
<style>
|
||||
/* tokens:begin Design tokens, the Lightwell design language (docs/design.md). This is the only place raw colors and sizes are defined. */
|
||||
:root {
|
||||
--accent:#1f5c5c; --accent-contrast:#ffffff;
|
||||
--bg:#f6f5f3; --surface:#ffffff; --surface-2:#efede9; --border:#e0ddd7; --border-strong:#c4c0b8;
|
||||
--text:#1d1c1a; --text-muted:#5f5b54; --text-faint:#7a766e;
|
||||
--accent-text:var(--accent);
|
||||
--accent-soft:color-mix(in srgb, var(--accent) 9%, var(--surface));
|
||||
--focus:color-mix(in srgb, var(--accent) 70%, #ffffff);
|
||||
--success:#1c6b3a; --success-bg:#e3f1e8; --warning:#85560a; --warning-bg:#f9eed6; --danger:#a3271e; --danger-bg:#f9e6e3; --neutral-bg:#eceae6;
|
||||
--s1:4px; --s2:8px; --s3:12px; --s4:16px; --s5:24px; --s6:32px; --s7:48px; --s8:64px;
|
||||
--radius:6px; --radius-lg:10px; --border-w:1px;
|
||||
--font:system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;
|
||||
--text-xs:12px; --text-sm:13px; --text-base:15px; --text-lg:18px; --text-xl:22px; --text-2xl:28px; --leading:1.5;
|
||||
--content-max:720px;
|
||||
color-scheme:light;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) { :root:not([data-theme="light"]) {
|
||||
--bg:#131312; --surface:#1b1b19; --surface-2:#252523; --border:#33322e; --border-strong:#4a4943;
|
||||
--text:#ecebe7; --text-muted:#adaaa2; --text-faint:#8f8c84;
|
||||
--accent-text:color-mix(in srgb, var(--accent) 45%, #ffffff);
|
||||
--accent-soft:color-mix(in srgb, var(--accent) 22%, var(--surface));
|
||||
--success:#86d3a0; --success-bg:#17301f; --warning:#e8bd6b; --warning-bg:#33280f; --danger:#f19a92; --danger-bg:#3a1b18; --neutral-bg:#2b2a27;
|
||||
color-scheme:dark;
|
||||
} }
|
||||
/* tokens:end */
|
||||
|
||||
*, *::before, *::after { box-sizing:border-box; }
|
||||
body { margin:0; background:var(--bg); color:var(--text); font:var(--text-base)/var(--leading) var(--font); -webkit-font-smoothing:antialiased; }
|
||||
h1, h2 { margin:0; line-height:1.25; font-weight:600; letter-spacing:-.01em; }
|
||||
h1 { font-size:var(--text-2xl); } h2 { font-size:var(--text-lg); margin:var(--s6) 0 var(--s3); }
|
||||
main { max-width:var(--content-max); margin:0 auto; padding:var(--s6) var(--s4) var(--s8); }
|
||||
.dim { color:var(--text-muted); } .small { font-size:var(--text-sm); } .num { font-variant-numeric:tabular-nums; }
|
||||
.banner { margin-top:var(--s4); padding:var(--s4) var(--s5); border-radius:var(--radius-lg); border:var(--border-w) solid var(--border); font-weight:600; background:var(--surface-2); }
|
||||
.banner.operational { background:var(--success-bg); color:var(--success); border-color:var(--success); }
|
||||
.banner.degraded { background:var(--warning-bg); color:var(--warning); border-color:var(--warning); }
|
||||
.banner.outage { background:var(--danger-bg); color:var(--danger); border-color:var(--danger); }
|
||||
.card { background:var(--surface); border:var(--border-w) solid var(--border); border-radius:var(--radius-lg); overflow:hidden; }
|
||||
.row { display:flex; align-items:center; justify-content:space-between; gap:var(--s3); flex-wrap:wrap; padding:var(--s3) var(--s4); border-bottom:var(--border-w) solid var(--border); }
|
||||
.row:last-child { border-bottom:0; }
|
||||
.badge { display:inline-flex; align-items:center; padding:2px var(--s2); border-radius:999px; font-size:var(--text-xs); font-weight:600; background:var(--neutral-bg); color:var(--text-muted); white-space:nowrap; }
|
||||
.badge-success { background:var(--success-bg); color:var(--success); }
|
||||
.badge-warning { background:var(--warning-bg); color:var(--warning); }
|
||||
.badge-danger { background:var(--danger-bg); color:var(--danger); }
|
||||
.inc { padding:var(--s4); border-bottom:var(--border-w) solid var(--border); }
|
||||
.inc:last-child { border-bottom:0; }
|
||||
.inc ul { margin:var(--s2) 0 0; padding-left:var(--s5); }
|
||||
:focus-visible { outline:2px solid var(--focus); outline-offset:2px; border-radius:var(--radius); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Status</h1>
|
||||
<div id="banner" class="banner" role="status">Loading</div>
|
||||
<h2>Services</h2>
|
||||
<div class="card" id="services"></div>
|
||||
<h2>Incidents</h2>
|
||||
<div class="card" id="incidents"></div>
|
||||
</main>
|
||||
<script>
|
||||
const $ = id => document.getElementById(id);
|
||||
const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; };
|
||||
const when = t => new Date(t * 1000).toLocaleString(undefined, {month:'short', day:'numeric', year:'numeric', hour:'numeric', minute:'2-digit'});
|
||||
const label = { operational: 'All systems are operational', degraded: 'Some systems are affected', outage: 'There is a service outage' };
|
||||
const statusBadge = s => s === 'up' ? el('span', 'badge badge-success', 'Operational') : s === 'down' ? el('span', 'badge badge-danger', 'Down') : el('span', 'badge', 'Waiting for first check');
|
||||
const incStatus = { investigating: 'Investigating', identified: 'Identified', monitoring: 'Monitoring', resolved: 'Resolved' };
|
||||
|
||||
async function load() {
|
||||
let d;
|
||||
try { d = await (await fetch('/status.json')).json(); } catch (e) { $('banner').textContent = 'The status could not be loaded. Try again in a moment.'; return; }
|
||||
$('banner').className = 'banner ' + d.overall;
|
||||
$('banner').textContent = label[d.overall] || d.overall;
|
||||
|
||||
const sv = $('services'); sv.replaceChildren();
|
||||
if (!d.monitors.length) sv.append(el('div', 'row dim', 'No services are published yet.'));
|
||||
for (const m of d.monitors) {
|
||||
const r = el('div', 'row');
|
||||
r.append(el('span', '', m.name));
|
||||
const right = el('span', 'small dim num');
|
||||
right.append(statusBadge(m.status), document.createTextNode(m.uptime >= 0 ? ' ' + m.uptime.toFixed(2) + '% uptime' : ''));
|
||||
r.append(right); sv.append(r);
|
||||
}
|
||||
|
||||
const ic = $('incidents'); ic.replaceChildren();
|
||||
if (!d.incidents.length) ic.append(el('div', 'row dim', 'No incidents in the last 14 days.'));
|
||||
for (const i of d.incidents) {
|
||||
const b = el('div', 'inc');
|
||||
const head = el('div'); head.append(el('strong', '', i.title + ' '), el('span', 'badge ' + (i.status === 'resolved' ? 'badge-success' : 'badge-warning'), incStatus[i.status] || i.status));
|
||||
b.append(head, el('div', 'small dim', 'Started ' + when(i.created) + (i.resolved ? ', resolved ' + when(i.resolved) : '')));
|
||||
if (i.updates.length) { const ul = el('ul', 'small'); for (const u of i.updates) ul.append(el('li', '', when(u.time) + ': ' + u.text)); b.append(ul); }
|
||||
ic.append(b);
|
||||
}
|
||||
}
|
||||
load(); setInterval(load, 30000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in new issue
Block a user