Initial commit: notifications, monitoring, on-call and incidents
One binary and one SQLite file. Topics with ntfy-style publish and subscribe, HTTP/TCP/DNS/TLS/heartbeat monitors with a public status page, schedules and escalation policies, and incidents with a timeline and post-mortem. The UI follows the Lightwell design language. Builds against an unreleased Reef (secretbox) through a local replace line. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
VeillaxandClaude Sonnet 5.5 committed
commit
0c5a57556a
22 files changed
+4953
No files matched your search
@@ -0,0 +1,327 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// On-call: who is responsible right now, and what happens to an alert nobody has
|
||||
// picked up. An alert follows its escalation policy, a list of steps, each of
|
||||
// which waits a delay and then notifies people and/or whoever is on call for a
|
||||
// schedule. Acknowledging or resolving the alert stops the walk. Alerts are kept
|
||||
// in the database with the time the next step is due, so a restart neither loses
|
||||
// nor repeats them.
|
||||
|
||||
type Override struct {
|
||||
UserID string `json:"userId"`
|
||||
From int64 `json:"from"`
|
||||
To int64 `json:"to"`
|
||||
}
|
||||
|
||||
// Schedule rotates through Members, handing over every RotationHours from Start.
|
||||
type Schedule struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Members []string `json:"members"` // user ids, in rotation order
|
||||
RotationHours int `json:"rotationHours"`
|
||||
Start int64 `json:"start"`
|
||||
Overrides []Override `json:"overrides"`
|
||||
}
|
||||
|
||||
// OnCall returns the user id on call at t, or "".
|
||||
func (sc *Schedule) OnCall(t time.Time) string {
|
||||
for i := len(sc.Overrides) - 1; i >= 0; i-- {
|
||||
o := sc.Overrides[i]
|
||||
if t.Unix() >= o.From && t.Unix() < o.To {
|
||||
return o.UserID
|
||||
}
|
||||
}
|
||||
if len(sc.Members) == 0 {
|
||||
return ""
|
||||
}
|
||||
rot := int64(sc.RotationHours) * 3600
|
||||
if rot <= 0 {
|
||||
return sc.Members[0]
|
||||
}
|
||||
elapsed := t.Unix() - sc.Start
|
||||
if elapsed < 0 {
|
||||
return sc.Members[0]
|
||||
}
|
||||
return sc.Members[(elapsed/rot)%int64(len(sc.Members))]
|
||||
}
|
||||
|
||||
type Step struct {
|
||||
DelayMin int `json:"delayMin"` // wait this long after the previous step (0 = at once)
|
||||
Users []string `json:"users"`
|
||||
Schedules []string `json:"schedules"`
|
||||
}
|
||||
|
||||
type Policy struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Steps []Step `json:"steps"`
|
||||
RepeatMin int `json:"repeatMin"` // after the last step, start over this many minutes later (0 = never)
|
||||
}
|
||||
|
||||
type Alert struct {
|
||||
ID string `json:"id"`
|
||||
Key string `json:"key"`
|
||||
Source string `json:"source"`
|
||||
Summary string `json:"summary"`
|
||||
Detail string `json:"detail"`
|
||||
Severity string `json:"severity"` // critical, warning, info
|
||||
Status string `json:"status"` // firing, acked, resolved
|
||||
PolicyID string `json:"policyId"`
|
||||
Step int `json:"step"`
|
||||
NextAt int64 `json:"nextAt"`
|
||||
Created int64 `json:"created"`
|
||||
Updated int64 `json:"updated"`
|
||||
Count int `json:"count"`
|
||||
AckedBy string `json:"ackedBy"`
|
||||
AckedAt int64 `json:"ackedAt"`
|
||||
SnoozeUntil int64 `json:"snoozeUntil"`
|
||||
ResolvedAt int64 `json:"resolvedAt"`
|
||||
IncidentID string `json:"incidentId"`
|
||||
Timeline []Event `json:"timeline,omitempty"`
|
||||
}
|
||||
|
||||
const alertCols = `id,key,source,summary,detail,severity,status,policy_id,step,next_at,created,updated,count,acked_by,acked_at,snooze_until,resolved_at,incident_id`
|
||||
|
||||
type scanner interface{ Scan(...any) error }
|
||||
|
||||
func scanAlert(r scanner) (*Alert, error) {
|
||||
var a Alert
|
||||
err := r.Scan(&a.ID, &a.Key, &a.Source, &a.Summary, &a.Detail, &a.Severity, &a.Status, &a.PolicyID, &a.Step, &a.NextAt,
|
||||
&a.Created, &a.Updated, &a.Count, &a.AckedBy, &a.AckedAt, &a.SnoozeUntil, &a.ResolvedAt, &a.IncidentID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, errNotFound
|
||||
}
|
||||
return &a, err
|
||||
}
|
||||
|
||||
func (s *Server) getAlert(id string) (*Alert, error) {
|
||||
return scanAlert(s.st.db.QueryRow(`SELECT `+alertCols+` FROM alerts WHERE id=?`, id))
|
||||
}
|
||||
|
||||
func (s *Server) listAlerts(status string, limit int) []Alert {
|
||||
q := `SELECT ` + alertCols + ` FROM alerts`
|
||||
var args []any
|
||||
switch status {
|
||||
case "open":
|
||||
q += ` WHERE status IN ('firing','acked')`
|
||||
case "firing", "acked", "resolved":
|
||||
q += ` WHERE status=?`
|
||||
args = append(args, status)
|
||||
}
|
||||
q += ` ORDER BY created DESC LIMIT ?`
|
||||
args = append(args, limit)
|
||||
out := []Alert{}
|
||||
rows, err := s.st.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
if a, err := scanAlert(rows); err == nil {
|
||||
out = append(out, *a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RaiseAlert opens an alert for key, or bumps the open one with the same key
|
||||
// (deduplication). It returns the alert and whether it is new.
|
||||
func (s *Server) RaiseAlert(key, source, summary, detail, severity, policyID string) (*Alert, bool, error) {
|
||||
now := s.now()
|
||||
if severity == "" {
|
||||
severity = "warning"
|
||||
}
|
||||
existing, err := scanAlert(s.st.db.QueryRow(`SELECT `+alertCols+` FROM alerts WHERE key=? AND status IN ('firing','acked') ORDER BY created DESC LIMIT 1`, key))
|
||||
if err == nil {
|
||||
existing.Count++
|
||||
existing.Updated = now.Unix()
|
||||
existing.Summary, existing.Detail = summary, detail
|
||||
_, err := s.st.db.Exec(`UPDATE alerts SET count=?, updated=?, summary=?, detail=? WHERE id=?`, existing.Count, existing.Updated, summary, detail, existing.ID)
|
||||
return existing, false, err
|
||||
}
|
||||
if !errors.Is(err, errNotFound) {
|
||||
return nil, false, err
|
||||
}
|
||||
a := &Alert{ID: newID(), Key: key, Source: source, Summary: summary, Detail: detail, Severity: severity, Status: "firing",
|
||||
PolicyID: policyID, NextAt: now.Unix(), Created: now.Unix(), Updated: now.Unix(), Count: 1}
|
||||
if policyID == "" {
|
||||
a.NextAt = 0 // nothing to escalate: it is only broadcast
|
||||
}
|
||||
_, err = s.st.db.Exec(`INSERT INTO alerts(`+alertCols+`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
a.ID, a.Key, a.Source, a.Summary, a.Detail, a.Severity, a.Status, a.PolicyID, a.Step, a.NextAt, a.Created, a.Updated, a.Count,
|
||||
a.AckedBy, a.AckedAt, a.SnoozeUntil, a.ResolvedAt, a.IncidentID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
s.st.addEvent("alert", a.ID, "created", source, summary, now)
|
||||
s.broadcast(Note{Status: "firing", Kind: "alert", Title: summary, Message: s.alertText(a), Severity: severity, Link: s.link("/#alerts"), Time: now.Unix()}, "alerts")
|
||||
return a, true, nil
|
||||
}
|
||||
|
||||
func (s *Server) alertText(a *Alert) string {
|
||||
t := strings.ToUpper(a.Severity) + ": " + a.Summary
|
||||
if a.Detail != "" {
|
||||
t += "\n" + a.Detail
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// ResolveAlertKey resolves the open alert for key, if any.
|
||||
func (s *Server) ResolveAlertKey(key, actor string) {
|
||||
var id string
|
||||
if s.st.db.QueryRow(`SELECT id FROM alerts WHERE key=? AND status IN ('firing','acked') LIMIT 1`, key).Scan(&id) == nil {
|
||||
s.ResolveAlert(id, actor)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) ResolveAlert(id, actor string) error {
|
||||
a, err := s.getAlert(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Status == "resolved" {
|
||||
return nil
|
||||
}
|
||||
now := s.now()
|
||||
if _, err := s.st.db.Exec(`UPDATE alerts SET status='resolved', next_at=0, resolved_at=?, updated=? WHERE id=?`, now.Unix(), now.Unix(), id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.st.addEvent("alert", id, "resolved", actor, "resolved", now)
|
||||
s.broadcast(Note{Status: "resolved", Kind: "alert", Title: a.Summary, Message: "Resolved: " + a.Summary, Severity: a.Severity, Link: s.link("/#alerts"), Time: now.Unix()}, "alerts")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) AckAlert(id, actor string) error {
|
||||
a, err := s.getAlert(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Status != "firing" {
|
||||
return errors.New("only a firing alert can be acknowledged")
|
||||
}
|
||||
now := s.now()
|
||||
if _, err := s.st.db.Exec(`UPDATE alerts SET status='acked', next_at=0, acked_by=?, acked_at=?, updated=? WHERE id=?`, actor, now.Unix(), now.Unix(), id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.st.addEvent("alert", id, "acked", actor, "acknowledged", now)
|
||||
s.broadcast(Note{Status: "acked", Kind: "alert", Title: a.Summary, Message: "Acknowledged by " + actor + ": " + a.Summary, Time: now.Unix()}, "alerts")
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnoozeAlert pauses escalation for d. When it ends the walk resumes at the step it was on.
|
||||
func (s *Server) SnoozeAlert(id, actor string, d time.Duration) error {
|
||||
a, err := s.getAlert(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Status != "firing" {
|
||||
return errors.New("only a firing alert can be snoozed")
|
||||
}
|
||||
until := s.now().Add(d).Unix()
|
||||
if _, err := s.st.db.Exec(`UPDATE alerts SET next_at=?, snooze_until=? WHERE id=?`, until, until, id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.st.addEvent("alert", id, "snoozed", actor, "snoozed for "+d.String(), s.now())
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------- escalation ----------
|
||||
|
||||
// escalate runs every alert step that has come due.
|
||||
func (s *Server) escalate(now time.Time) {
|
||||
rows, err := s.st.db.Query(`SELECT id FROM alerts WHERE status='firing' AND next_at>0 AND next_at<=?`, now.Unix())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if rows.Scan(&id) == nil {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
for _, id := range ids {
|
||||
s.runStep(id, now)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) runStep(id string, now time.Time) {
|
||||
a, err := s.getAlert(id)
|
||||
if err != nil || a.Status != "firing" {
|
||||
return
|
||||
}
|
||||
pol, err := getOf[Policy](s.st, "policy", a.PolicyID)
|
||||
if err != nil || len(pol.Steps) == 0 {
|
||||
s.st.db.Exec(`UPDATE alerts SET next_at=0 WHERE id=?`, id) // policy deleted: stop trying
|
||||
s.st.addEvent("alert", id, "note", "sandbar", "its escalation policy is gone or empty", now)
|
||||
return
|
||||
}
|
||||
if a.Step >= len(pol.Steps) {
|
||||
a.Step = 0
|
||||
}
|
||||
step := pol.Steps[a.Step]
|
||||
var people []string
|
||||
seen := map[string]bool{}
|
||||
add := func(uid string) {
|
||||
if uid != "" && !seen[uid] {
|
||||
seen[uid] = true
|
||||
people = append(people, uid)
|
||||
}
|
||||
}
|
||||
for _, u := range step.Users {
|
||||
add(u)
|
||||
}
|
||||
for _, scID := range step.Schedules {
|
||||
if sc, err := getOf[Schedule](s.st, "schedule", scID); err == nil {
|
||||
add(sc.OnCall(now))
|
||||
}
|
||||
}
|
||||
var names []string
|
||||
for _, uid := range people {
|
||||
if u, err := s.userByID(uid); err == nil {
|
||||
s.notifyUser(u, Note{Status: "firing", Kind: "alert", Title: a.Summary, Message: s.alertText(a), Severity: a.Severity,
|
||||
Link: s.link("/#alerts"), Time: now.Unix()})
|
||||
names = append(names, u.Name)
|
||||
}
|
||||
}
|
||||
text := fmt.Sprintf("step %d: notified %s", a.Step+1, strings.Join(names, ", "))
|
||||
if len(names) == 0 {
|
||||
text = fmt.Sprintf("step %d: nobody to notify (empty schedule or removed users)", a.Step+1)
|
||||
}
|
||||
s.st.addEvent("alert", id, "escalated", "sandbar", text, now)
|
||||
|
||||
next := a.Step + 1
|
||||
var nextAt int64
|
||||
switch {
|
||||
case next < len(pol.Steps):
|
||||
nextAt = now.Add(time.Duration(pol.Steps[next].DelayMin) * time.Minute).Unix()
|
||||
case pol.RepeatMin > 0:
|
||||
next = 0
|
||||
nextAt = now.Add(time.Duration(pol.RepeatMin) * time.Minute).Unix()
|
||||
}
|
||||
s.st.db.Exec(`UPDATE alerts SET step=?, next_at=?, updated=? WHERE id=?`, next, nextAt, now.Unix(), id)
|
||||
}
|
||||
|
||||
// notifyUser reaches one person: a message on their personal topic (u-<id>, which
|
||||
// their phone subscribes to) and an email if the server can send mail.
|
||||
func (s *Server) notifyUser(u *User, n Note) {
|
||||
s.Publish(userTopic(u.ID), n.subject(), n.text(), n.priority(), n.tags())
|
||||
if u.Email != "" && (s.opts.SMTP.Host != "" || s.opts.Mailer != nil) {
|
||||
go func() {
|
||||
if err := s.Mail(u.Email, n.subject(), n.text()); err != nil {
|
||||
s.st.addEvent("alert", "", "error", "sandbar", "email to "+u.Email+": "+err.Error(), s.now())
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func userTopic(id string) string { return "u-" + id }
|
||||
Reference in new issue
Block a user