package sandbar import ( "errors" "fmt" "strings" "time" ) // Incidents are the human side: something is wrong and people are working on it. // An incident has a severity, a status that moves forward, a lead and a comms // owner, a timeline of everything that happened, and a post-mortem. Every change // is announced on the "incidents" topic and the broadcast channels. type Incident struct { ID string `json:"id"` Title string `json:"title"` Severity string `json:"severity"` // sev1 (worst) to sev3 Status string `json:"status"` // investigating, identified, monitoring, resolved Lead string `json:"lead"` // user id Comms string `json:"comms"` // user id Summary string `json:"summary"` Postmortem string `json:"postmortem"` Public bool `json:"public"` // shown on the status page Components []string `json:"components"` // monitor ids the status page says are affected Created int64 `json:"created"` Resolved int64 `json:"resolved"` AlertIDs []string `json:"alertIds"` Timeline []Event `json:"timeline,omitempty"` } var incidentStatuses = []string{"investigating", "identified", "monitoring", "resolved"} func validStatus(s string) bool { for _, v := range incidentStatuses { if v == s { return true } } return false } func validSeverity(s string) bool { return s == "sev1" || s == "sev2" || s == "sev3" } func (s *Server) userName(id string) string { if u, err := s.userByID(id); err == nil { return u.Name } return "" } func (s *Server) announceIncident(in *Incident, status, text string) { s.broadcast(Note{Status: status, Kind: "incident", Title: in.Title, Message: fmt.Sprintf("[%s] %s: %s", strings.ToUpper(in.Severity), in.Status, text), Severity: map[string]string{"sev1": "critical", "sev2": "warning"}[in.Severity], Link: s.link("/#incidents"), Time: s.now().Unix()}, "incidents") } // DeclareIncident opens an incident, optionally linking the alerts that caused it. func (s *Server) DeclareIncident(title, severity, summary, lead, actor string, alertIDs []string) (*Incident, error) { title = strings.TrimSpace(title) if title == "" { return nil, errors.New("give the incident a title") } if severity == "" { severity = "sev2" } if !validSeverity(severity) { return nil, errors.New("severity must be sev1, sev2 or sev3") } now := s.now() in := &Incident{ID: newID(), Title: title, Severity: severity, Status: "investigating", Lead: lead, Summary: summary, Created: now.Unix(), AlertIDs: []string{}, Components: []string{}} for _, id := range alertIDs { if a, err := s.getAlert(id); err == nil { in.AlertIDs = append(in.AlertIDs, a.ID) s.st.db.Exec(`UPDATE alerts SET incident_id=? WHERE id=?`, in.ID, a.ID) s.st.addEvent("alert", a.ID, "incident", actor, "linked to incident "+title, now) } } if err := s.st.put("incident", in.ID, in); err != nil { return nil, err } s.st.addEvent("incident", in.ID, "declared", actor, fmt.Sprintf("declared %s: %s", severity, title), now) if lead != "" { s.st.addEvent("incident", in.ID, "role", actor, "lead: "+s.userName(lead), now) } s.announceIncident(in, "declared", firstNonEmpty(summary, "incident declared")) return in, nil } func firstNonEmpty(v ...string) string { for _, s := range v { if s != "" { return s } } return "" } // IncidentPatch holds the fields an update may change; nil means unchanged. type IncidentPatch struct { Title *string `json:"title"` Severity *string `json:"severity"` Status *string `json:"status"` Lead *string `json:"lead"` Comms *string `json:"comms"` Summary *string `json:"summary"` Postmortem *string `json:"postmortem"` Public *bool `json:"public"` Components *[]string `json:"components"` Note string `json:"note"` // text for the update announcement and timeline } func (s *Server) UpdateIncident(id, actor string, p IncidentPatch) (*Incident, error) { in, err := getOf[Incident](s.st, "incident", id) if err != nil { return nil, err } now := s.now() prevStatus := in.Status var changes []string if p.Severity != nil && *p.Severity != in.Severity { if !validSeverity(*p.Severity) { return nil, errors.New("severity must be sev1, sev2 or sev3") } changes = append(changes, "severity "+in.Severity+" to "+*p.Severity) in.Severity = *p.Severity } if p.Status != nil && *p.Status != in.Status { if !validStatus(*p.Status) { return nil, errors.New("status must be investigating, identified, monitoring or resolved") } changes = append(changes, "status "+in.Status+" to "+*p.Status) in.Status = *p.Status if in.Status == "resolved" { in.Resolved = now.Unix() } else { in.Resolved = 0 } } if p.Lead != nil && *p.Lead != in.Lead { in.Lead = *p.Lead changes = append(changes, "lead: "+s.userName(in.Lead)) } if p.Comms != nil && *p.Comms != in.Comms { in.Comms = *p.Comms changes = append(changes, "comms: "+s.userName(in.Comms)) } if p.Title != nil && strings.TrimSpace(*p.Title) != "" && *p.Title != in.Title { in.Title = strings.TrimSpace(*p.Title) changes = append(changes, "renamed") } if p.Summary != nil { in.Summary = *p.Summary } if p.Postmortem != nil { in.Postmortem = *p.Postmortem } wasPublic, statusChanged := in.Public, in.Status != prevStatus if p.Public != nil { in.Public = *p.Public } if p.Components != nil { in.Components = cleanIDs(*p.Components) } if err := s.st.put("incident", in.ID, in); err != nil { return nil, err } s.postToStatusPage(in, wasPublic, statusChanged, p.Note, now) if len(changes) > 0 || p.Note != "" { text := strings.Join(changes, "; ") if p.Note != "" { text = strings.TrimSpace(text + ": " + p.Note) } s.st.addEvent("incident", in.ID, "update", actor, text, now) kind := "update" if in.Status == "resolved" { kind = "resolved" } s.announceIncident(in, kind, text) } return in, nil } func (s *Server) CommentIncident(id, actor, text string) error { if _, err := getOf[Incident](s.st, "incident", id); err != nil { return err } if strings.TrimSpace(text) == "" { return errors.New("write something") } s.st.addEvent("incident", id, "comment", actor, text, s.now()) return nil } // postmortemTemplate is a starting point, filled with what the timeline already knows. func (s *Server) postmortemTemplate(in *Incident) string { var b strings.Builder fmt.Fprintf(&b, "# Post-mortem: %s\n\n", in.Title) fmt.Fprintf(&b, "Severity: %s\nStarted: %s\n", in.Severity, time.Unix(in.Created, 0).UTC().Format(time.RFC1123)) if in.Resolved > 0 { fmt.Fprintf(&b, "Resolved: %s (%s)\n", time.Unix(in.Resolved, 0).UTC().Format(time.RFC1123), time.Duration(in.Resolved-in.Created)*time.Second) } b.WriteString("\n## Summary\n\n\n## Impact\n\n\n## Root cause\n\n\n## Timeline\n\n") for _, e := range s.st.events("incident", in.ID) { fmt.Fprintf(&b, "- %s %s\n", time.Unix(e.Time, 0).UTC().Format("15:04:05"), e.Text) } b.WriteString("\n## What went well\n\n\n## What went badly\n\n\n## Action items\n\n") return b.String() }