package sandbar import ( "database/sql" "errors" "fmt" "strings" "time" ) // On-call: who is responsible right now, and what happens to an alert nobody has // picked up. An alert follows its escalation policy, a list of steps, each of // which waits a delay and then notifies people and/or whoever is on call for a // schedule. Acknowledging or resolving the alert stops the walk. Alerts are kept // in the database with the time the next step is due, so a restart neither loses // nor repeats them. type Override struct { UserID string `json:"userId"` From int64 `json:"from"` To int64 `json:"to"` } // Schedule rotates through Members, handing over every RotationHours from Start. type Schedule struct { ID string `json:"id"` Name string `json:"name"` Members []string `json:"members"` // user ids, in rotation order RotationHours int `json:"rotationHours"` Start int64 `json:"start"` Overrides []Override `json:"overrides"` } // OnCall returns the user id on call at t, or "". func (sc *Schedule) OnCall(t time.Time) string { for i := len(sc.Overrides) - 1; i >= 0; i-- { o := sc.Overrides[i] if t.Unix() >= o.From && t.Unix() < o.To { return o.UserID } } if len(sc.Members) == 0 { return "" } rot := int64(sc.RotationHours) * 3600 if rot <= 0 { return sc.Members[0] } elapsed := t.Unix() - sc.Start if elapsed < 0 { return sc.Members[0] } return sc.Members[(elapsed/rot)%int64(len(sc.Members))] } type Step struct { DelayMin int `json:"delayMin"` // wait this long after the previous step (0 = at once) Users []string `json:"users"` Schedules []string `json:"schedules"` } type Policy struct { ID string `json:"id"` Name string `json:"name"` Steps []Step `json:"steps"` RepeatMin int `json:"repeatMin"` // after the last step, start over this many minutes later (0 = never) } type Alert struct { ID string `json:"id"` Key string `json:"key"` Source string `json:"source"` Summary string `json:"summary"` Detail string `json:"detail"` Severity string `json:"severity"` // critical, warning, info Status string `json:"status"` // firing, acked, resolved PolicyID string `json:"policyId"` Step int `json:"step"` NextAt int64 `json:"nextAt"` Created int64 `json:"created"` Updated int64 `json:"updated"` Count int `json:"count"` AckedBy string `json:"ackedBy"` AckedAt int64 `json:"ackedAt"` SnoozeUntil int64 `json:"snoozeUntil"` ResolvedAt int64 `json:"resolvedAt"` IncidentID string `json:"incidentId"` Timeline []Event `json:"timeline,omitempty"` } const alertCols = `id,key,source,summary,detail,severity,status,policy_id,step,next_at,created,updated,count,acked_by,acked_at,snooze_until,resolved_at,incident_id` type scanner interface{ Scan(...any) error } func scanAlert(r scanner) (*Alert, error) { var a Alert err := r.Scan(&a.ID, &a.Key, &a.Source, &a.Summary, &a.Detail, &a.Severity, &a.Status, &a.PolicyID, &a.Step, &a.NextAt, &a.Created, &a.Updated, &a.Count, &a.AckedBy, &a.AckedAt, &a.SnoozeUntil, &a.ResolvedAt, &a.IncidentID) if errors.Is(err, sql.ErrNoRows) { return nil, errNotFound } return &a, err } func (s *Server) getAlert(id string) (*Alert, error) { return scanAlert(s.st.db.QueryRow(`SELECT `+alertCols+` FROM alerts WHERE id=?`, id)) } func (s *Server) listAlerts(status string, limit int) []Alert { q := `SELECT ` + alertCols + ` FROM alerts` var args []any switch status { case "open": q += ` WHERE status IN ('firing','acked')` case "firing", "acked", "resolved": q += ` WHERE status=?` args = append(args, status) } q += ` ORDER BY created DESC LIMIT ?` args = append(args, limit) out := []Alert{} rows, err := s.st.db.Query(q, args...) if err != nil { return out } defer rows.Close() for rows.Next() { if a, err := scanAlert(rows); err == nil { out = append(out, *a) } } return out } // RaiseAlert opens an alert for key, or bumps the open one with the same key // (deduplication). It returns the alert and whether it is new. func (s *Server) RaiseAlert(key, source, summary, detail, severity, policyID string) (*Alert, bool, error) { now := s.now() if severity == "" { severity = "warning" } existing, err := scanAlert(s.st.db.QueryRow(`SELECT `+alertCols+` FROM alerts WHERE key=? AND status IN ('firing','acked') ORDER BY created DESC LIMIT 1`, key)) if err == nil { existing.Count++ existing.Updated = now.Unix() existing.Summary, existing.Detail = summary, detail _, err := s.st.db.Exec(`UPDATE alerts SET count=?, updated=?, summary=?, detail=? WHERE id=?`, existing.Count, existing.Updated, summary, detail, existing.ID) return existing, false, err } if !errors.Is(err, errNotFound) { return nil, false, err } a := &Alert{ID: newID(), Key: key, Source: source, Summary: summary, Detail: detail, Severity: severity, Status: "firing", PolicyID: policyID, NextAt: now.Unix(), Created: now.Unix(), Updated: now.Unix(), Count: 1} if policyID == "" { a.NextAt = 0 // nothing to escalate: it is only broadcast } _, err = s.st.db.Exec(`INSERT INTO alerts(`+alertCols+`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, a.ID, a.Key, a.Source, a.Summary, a.Detail, a.Severity, a.Status, a.PolicyID, a.Step, a.NextAt, a.Created, a.Updated, a.Count, a.AckedBy, a.AckedAt, a.SnoozeUntil, a.ResolvedAt, a.IncidentID) if err != nil { return nil, false, err } s.st.addEvent("alert", a.ID, "created", source, summary, now) s.broadcast(Note{Status: "firing", Kind: "alert", Title: summary, Message: s.alertText(a), Severity: severity, Link: s.link("/#alerts"), Time: now.Unix()}, "alerts") return a, true, nil } func (s *Server) alertText(a *Alert) string { t := strings.ToUpper(a.Severity) + ": " + a.Summary if a.Detail != "" { t += "\n" + a.Detail } return t } // ResolveAlertKey resolves the open alert for key, if any. func (s *Server) ResolveAlertKey(key, actor string) { var id string if s.st.db.QueryRow(`SELECT id FROM alerts WHERE key=? AND status IN ('firing','acked') LIMIT 1`, key).Scan(&id) == nil { s.ResolveAlert(id, actor) } } func (s *Server) ResolveAlert(id, actor string) error { a, err := s.getAlert(id) if err != nil { return err } if a.Status == "resolved" { return nil } now := s.now() if _, err := s.st.db.Exec(`UPDATE alerts SET status='resolved', next_at=0, resolved_at=?, updated=? WHERE id=?`, now.Unix(), now.Unix(), id); err != nil { return err } s.st.addEvent("alert", id, "resolved", actor, "resolved", now) s.broadcast(Note{Status: "resolved", Kind: "alert", Title: a.Summary, Message: "Resolved: " + a.Summary, Severity: a.Severity, Link: s.link("/#alerts"), Time: now.Unix()}, "alerts") return nil } func (s *Server) AckAlert(id, actor string) error { a, err := s.getAlert(id) if err != nil { return err } if a.Status != "firing" { return errors.New("only a firing alert can be acknowledged") } now := s.now() if _, err := s.st.db.Exec(`UPDATE alerts SET status='acked', next_at=0, acked_by=?, acked_at=?, updated=? WHERE id=?`, actor, now.Unix(), now.Unix(), id); err != nil { return err } s.st.addEvent("alert", id, "acked", actor, "acknowledged", now) s.broadcast(Note{Status: "acked", Kind: "alert", Title: a.Summary, Message: "Acknowledged by " + actor + ": " + a.Summary, Time: now.Unix()}, "alerts") return nil } // SnoozeAlert pauses escalation for d. When it ends the walk resumes at the step it was on. func (s *Server) SnoozeAlert(id, actor string, d time.Duration) error { a, err := s.getAlert(id) if err != nil { return err } if a.Status != "firing" { return errors.New("only a firing alert can be snoozed") } until := s.now().Add(d).Unix() if _, err := s.st.db.Exec(`UPDATE alerts SET next_at=?, snooze_until=? WHERE id=?`, until, until, id); err != nil { return err } s.st.addEvent("alert", id, "snoozed", actor, "snoozed for "+d.String(), s.now()) return nil } // ---------- escalation ---------- // escalate runs every alert step that has come due. func (s *Server) escalate(now time.Time) { rows, err := s.st.db.Query(`SELECT id FROM alerts WHERE status='firing' AND next_at>0 AND next_at<=?`, now.Unix()) if err != nil { return } var ids []string for rows.Next() { var id string if rows.Scan(&id) == nil { ids = append(ids, id) } } rows.Close() for _, id := range ids { s.runStep(id, now) } } func (s *Server) runStep(id string, now time.Time) { a, err := s.getAlert(id) if err != nil || a.Status != "firing" { return } pol, err := getOf[Policy](s.st, "policy", a.PolicyID) if err != nil || len(pol.Steps) == 0 { s.st.db.Exec(`UPDATE alerts SET next_at=0 WHERE id=?`, id) // policy deleted: stop trying s.st.addEvent("alert", id, "note", "sandbar", "its escalation policy is gone or empty", now) return } if a.Step >= len(pol.Steps) { a.Step = 0 } step := pol.Steps[a.Step] var people []string seen := map[string]bool{} add := func(uid string) { if uid != "" && !seen[uid] { seen[uid] = true people = append(people, uid) } } for _, u := range step.Users { add(u) } for _, scID := range step.Schedules { if sc, err := getOf[Schedule](s.st, "schedule", scID); err == nil { add(sc.OnCall(now)) } } var names []string for _, uid := range people { if u, err := s.userByID(uid); err == nil { s.notifyUser(u, Note{Status: "firing", Kind: "alert", Title: a.Summary, Message: s.alertText(a), Severity: a.Severity, Link: s.link("/#alerts"), Time: now.Unix()}) names = append(names, u.Name) } } text := fmt.Sprintf("step %d: notified %s", a.Step+1, strings.Join(names, ", ")) if len(names) == 0 { text = fmt.Sprintf("step %d: nobody to notify (empty schedule or removed users)", a.Step+1) } s.st.addEvent("alert", id, "escalated", "sandbar", text, now) next := a.Step + 1 var nextAt int64 switch { case next < len(pol.Steps): nextAt = now.Add(time.Duration(pol.Steps[next].DelayMin) * time.Minute).Unix() case pol.RepeatMin > 0: next = 0 nextAt = now.Add(time.Duration(pol.RepeatMin) * time.Minute).Unix() } s.st.db.Exec(`UPDATE alerts SET step=?, next_at=?, updated=? WHERE id=?`, next, nextAt, now.Unix(), id) } // notifyUser reaches one person: a message on their personal topic (u-, which // their phone subscribes to) and an email if the server can send mail. func (s *Server) notifyUser(u *User, n Note) { s.Publish(userTopic(u.ID), n.subject(), n.text(), n.priority(), n.tags()) if u.Email != "" && (s.opts.SMTP.Host != "" || s.opts.Mailer != nil) { go func() { if err := s.Mail(u.Email, n.subject(), n.text()); err != nil { s.st.addEvent("alert", "", "error", "sandbar", "email to "+u.Email+": "+err.Error(), s.now()) } }() } } func userTopic(id string) string { return "u-" + id }