Initial commit: notifications, monitoring, on-call and incidents
One binary and one SQLite file. Topics with ntfy-style publish and subscribe, HTTP/TCP/DNS/TLS/heartbeat monitors with a public status page, schedules and escalation policies, and incidents with a timeline and post-mortem. The UI follows the Lightwell design language. Builds against an unreleased Reef (secretbox) through a local replace line. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
VeillaxandClaude Sonnet 5.5 committed
commit
0c5a57556a
22 files changed
+4953
No files matched your search
@@ -0,0 +1,200 @@
|
||||
package sandbar
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Incidents are the human side: something is wrong and people are working on it.
|
||||
// An incident has a severity, a status that moves forward, a lead and a comms
|
||||
// owner, a timeline of everything that happened, and a post-mortem. Every change
|
||||
// is announced on the "incidents" topic and the broadcast channels.
|
||||
|
||||
type Incident struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Severity string `json:"severity"` // sev1 (worst) to sev3
|
||||
Status string `json:"status"` // investigating, identified, monitoring, resolved
|
||||
Lead string `json:"lead"` // user id
|
||||
Comms string `json:"comms"` // user id
|
||||
Summary string `json:"summary"`
|
||||
Postmortem string `json:"postmortem"`
|
||||
Public bool `json:"public"` // shown on the status page
|
||||
Created int64 `json:"created"`
|
||||
Resolved int64 `json:"resolved"`
|
||||
AlertIDs []string `json:"alertIds"`
|
||||
Timeline []Event `json:"timeline,omitempty"`
|
||||
}
|
||||
|
||||
var incidentStatuses = []string{"investigating", "identified", "monitoring", "resolved"}
|
||||
|
||||
func validStatus(s string) bool {
|
||||
for _, v := range incidentStatuses {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func validSeverity(s string) bool { return s == "sev1" || s == "sev2" || s == "sev3" }
|
||||
|
||||
func (s *Server) userName(id string) string {
|
||||
if u, err := s.userByID(id); err == nil {
|
||||
return u.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Server) announceIncident(in *Incident, status, text string) {
|
||||
s.broadcast(Note{Status: status, Kind: "incident", Title: in.Title, Message: fmt.Sprintf("[%s] %s: %s", strings.ToUpper(in.Severity), in.Status, text),
|
||||
Severity: map[string]string{"sev1": "critical", "sev2": "warning"}[in.Severity], Link: s.link("/#incidents"), Time: s.now().Unix()}, "incidents")
|
||||
}
|
||||
|
||||
// DeclareIncident opens an incident, optionally linking the alerts that caused it.
|
||||
func (s *Server) DeclareIncident(title, severity, summary, lead, actor string, alertIDs []string) (*Incident, error) {
|
||||
title = strings.TrimSpace(title)
|
||||
if title == "" {
|
||||
return nil, errors.New("give the incident a title")
|
||||
}
|
||||
if severity == "" {
|
||||
severity = "sev2"
|
||||
}
|
||||
if !validSeverity(severity) {
|
||||
return nil, errors.New("severity must be sev1, sev2 or sev3")
|
||||
}
|
||||
now := s.now()
|
||||
in := &Incident{ID: newID(), Title: title, Severity: severity, Status: "investigating", Lead: lead, Summary: summary, Created: now.Unix(), AlertIDs: []string{}}
|
||||
for _, id := range alertIDs {
|
||||
if a, err := s.getAlert(id); err == nil {
|
||||
in.AlertIDs = append(in.AlertIDs, a.ID)
|
||||
s.st.db.Exec(`UPDATE alerts SET incident_id=? WHERE id=?`, in.ID, a.ID)
|
||||
s.st.addEvent("alert", a.ID, "incident", actor, "linked to incident "+title, now)
|
||||
}
|
||||
}
|
||||
if err := s.st.put("incident", in.ID, in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.st.addEvent("incident", in.ID, "declared", actor, fmt.Sprintf("declared %s: %s", severity, title), now)
|
||||
if lead != "" {
|
||||
s.st.addEvent("incident", in.ID, "role", actor, "lead: "+s.userName(lead), now)
|
||||
}
|
||||
s.announceIncident(in, "declared", firstNonEmpty(summary, "incident declared"))
|
||||
return in, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(v ...string) string {
|
||||
for _, s := range v {
|
||||
if s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// IncidentPatch holds the fields an update may change; nil means unchanged.
|
||||
type IncidentPatch struct {
|
||||
Title *string `json:"title"`
|
||||
Severity *string `json:"severity"`
|
||||
Status *string `json:"status"`
|
||||
Lead *string `json:"lead"`
|
||||
Comms *string `json:"comms"`
|
||||
Summary *string `json:"summary"`
|
||||
Postmortem *string `json:"postmortem"`
|
||||
Public *bool `json:"public"`
|
||||
Note string `json:"note"` // text for the update announcement and timeline
|
||||
}
|
||||
|
||||
func (s *Server) UpdateIncident(id, actor string, p IncidentPatch) (*Incident, error) {
|
||||
in, err := getOf[Incident](s.st, "incident", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
var changes []string
|
||||
if p.Severity != nil && *p.Severity != in.Severity {
|
||||
if !validSeverity(*p.Severity) {
|
||||
return nil, errors.New("severity must be sev1, sev2 or sev3")
|
||||
}
|
||||
changes = append(changes, "severity "+in.Severity+" to "+*p.Severity)
|
||||
in.Severity = *p.Severity
|
||||
}
|
||||
if p.Status != nil && *p.Status != in.Status {
|
||||
if !validStatus(*p.Status) {
|
||||
return nil, errors.New("status must be investigating, identified, monitoring or resolved")
|
||||
}
|
||||
changes = append(changes, "status "+in.Status+" to "+*p.Status)
|
||||
in.Status = *p.Status
|
||||
if in.Status == "resolved" {
|
||||
in.Resolved = now.Unix()
|
||||
} else {
|
||||
in.Resolved = 0
|
||||
}
|
||||
}
|
||||
if p.Lead != nil && *p.Lead != in.Lead {
|
||||
in.Lead = *p.Lead
|
||||
changes = append(changes, "lead: "+s.userName(in.Lead))
|
||||
}
|
||||
if p.Comms != nil && *p.Comms != in.Comms {
|
||||
in.Comms = *p.Comms
|
||||
changes = append(changes, "comms: "+s.userName(in.Comms))
|
||||
}
|
||||
if p.Title != nil && strings.TrimSpace(*p.Title) != "" && *p.Title != in.Title {
|
||||
in.Title = strings.TrimSpace(*p.Title)
|
||||
changes = append(changes, "renamed")
|
||||
}
|
||||
if p.Summary != nil {
|
||||
in.Summary = *p.Summary
|
||||
}
|
||||
if p.Postmortem != nil {
|
||||
in.Postmortem = *p.Postmortem
|
||||
}
|
||||
if p.Public != nil {
|
||||
in.Public = *p.Public
|
||||
}
|
||||
if err := s.st.put("incident", in.ID, in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(changes) > 0 || p.Note != "" {
|
||||
text := strings.Join(changes, "; ")
|
||||
if p.Note != "" {
|
||||
text = strings.TrimSpace(text + ": " + p.Note)
|
||||
}
|
||||
s.st.addEvent("incident", in.ID, "update", actor, text, now)
|
||||
kind := "update"
|
||||
if in.Status == "resolved" {
|
||||
kind = "resolved"
|
||||
}
|
||||
s.announceIncident(in, kind, text)
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
func (s *Server) CommentIncident(id, actor, text string) error {
|
||||
if _, err := getOf[Incident](s.st, "incident", id); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return errors.New("write something")
|
||||
}
|
||||
s.st.addEvent("incident", id, "comment", actor, text, s.now())
|
||||
return nil
|
||||
}
|
||||
|
||||
// postmortemTemplate is a starting point, filled with what the timeline already knows.
|
||||
func (s *Server) postmortemTemplate(in *Incident) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "# Post-mortem: %s\n\n", in.Title)
|
||||
fmt.Fprintf(&b, "Severity: %s\nStarted: %s\n", in.Severity, time.Unix(in.Created, 0).UTC().Format(time.RFC1123))
|
||||
if in.Resolved > 0 {
|
||||
fmt.Fprintf(&b, "Resolved: %s (%s)\n", time.Unix(in.Resolved, 0).UTC().Format(time.RFC1123), time.Duration(in.Resolved-in.Created)*time.Second)
|
||||
}
|
||||
b.WriteString("\n## Summary\n\n\n## Impact\n\n\n## Root cause\n\n\n## Timeline\n\n")
|
||||
for _, e := range s.st.events("incident", in.ID) {
|
||||
fmt.Fprintf(&b, "- %s %s\n", time.Unix(e.Time, 0).UTC().Format("15:04:05"), e.Text)
|
||||
}
|
||||
b.WriteString("\n## What went well\n\n\n## What went badly\n\n\n## Action items\n\n")
|
||||
return b.String()
|
||||
}
|
||||
Reference in new issue
Block a user