Sandbar/internal/sandbar/incident.go

package sandbar

import (
	"errors"
	"fmt"
	"strings"
	"time"
)

// Incidents are the human side: something is wrong and people are working on it.
// An incident has a severity, a status that moves forward, a lead and a comms
// owner, a timeline of everything that happened, and a post-mortem. Every change
// is announced on the "incidents" topic and the broadcast channels.

type Incident struct {
	ID         string   `json:"id"`
	Title      string   `json:"title"`
	Severity   string   `json:"severity"` // sev1 (worst) to sev3
	Status     string   `json:"status"`   // investigating, identified, monitoring, resolved
	Lead       string   `json:"lead"`     // user id
	Comms      string   `json:"comms"`    // user id
	Summary    string   `json:"summary"`
	Postmortem string   `json:"postmortem"`
	Public     bool     `json:"public"`     // shown on the status page
	Components []string `json:"components"` // monitor ids the status page says are affected
	Created    int64    `json:"created"`
	Resolved   int64    `json:"resolved"`
	AlertIDs   []string `json:"alertIds"`
	Timeline   []Event  `json:"timeline,omitempty"`
}

var incidentStatuses = []string{"investigating", "identified", "monitoring", "resolved"}

func validStatus(s string) bool {
	for _, v := range incidentStatuses {
		if v == s {
			return true
		}
	}
	return false
}

func validSeverity(s string) bool { return s == "sev1" || s == "sev2" || s == "sev3" }

func (s *Server) userName(id string) string {
	if u, err := s.userByID(id); err == nil {
		return u.Name
	}
	return ""
}

func (s *Server) announceIncident(in *Incident, status, text string) {
	s.broadcast(Note{Status: status, Kind: "incident", Title: in.Title, Message: fmt.Sprintf("[%s] %s: %s", strings.ToUpper(in.Severity), in.Status, text),
		Severity: map[string]string{"sev1": "critical", "sev2": "warning"}[in.Severity], Link: s.link("/#incidents"), Time: s.now().Unix()}, "incidents")
}

// DeclareIncident opens an incident, optionally linking the alerts that caused it.
func (s *Server) DeclareIncident(title, severity, summary, lead, actor string, alertIDs []string) (*Incident, error) {
	title = strings.TrimSpace(title)
	if title == "" {
		return nil, errors.New("give the incident a title")
	}
	if severity == "" {
		severity = "sev2"
	}
	if !validSeverity(severity) {
		return nil, errors.New("severity must be sev1, sev2 or sev3")
	}
	now := s.now()
	in := &Incident{ID: newID(), Title: title, Severity: severity, Status: "investigating", Lead: lead, Summary: summary, Created: now.Unix(), AlertIDs: []string{}, Components: []string{}}
	for _, id := range alertIDs {
		if a, err := s.getAlert(id); err == nil {
			in.AlertIDs = append(in.AlertIDs, a.ID)
			s.st.db.Exec(`UPDATE alerts SET incident_id=? WHERE id=?`, in.ID, a.ID)
			s.st.addEvent("alert", a.ID, "incident", actor, "linked to incident "+title, now)
		}
	}
	if err := s.st.put("incident", in.ID, in); err != nil {
		return nil, err
	}
	s.st.addEvent("incident", in.ID, "declared", actor, fmt.Sprintf("declared %s: %s", severity, title), now)
	if lead != "" {
		s.st.addEvent("incident", in.ID, "role", actor, "lead: "+s.userName(lead), now)
	}
	s.announceIncident(in, "declared", firstNonEmpty(summary, "incident declared"))
	return in, nil
}

func firstNonEmpty(v ...string) string {
	for _, s := range v {
		if s != "" {
			return s
		}
	}
	return ""
}

// IncidentPatch holds the fields an update may change; nil means unchanged.
type IncidentPatch struct {
	Title      *string   `json:"title"`
	Severity   *string   `json:"severity"`
	Status     *string   `json:"status"`
	Lead       *string   `json:"lead"`
	Comms      *string   `json:"comms"`
	Summary    *string   `json:"summary"`
	Postmortem *string   `json:"postmortem"`
	Public     *bool     `json:"public"`
	Components *[]string `json:"components"`
	Note       string    `json:"note"` // text for the update announcement and timeline
}

func (s *Server) UpdateIncident(id, actor string, p IncidentPatch) (*Incident, error) {
	in, err := getOf[Incident](s.st, "incident", id)
	if err != nil {
		return nil, err
	}
	now := s.now()
	prevStatus := in.Status
	var changes []string
	if p.Severity != nil && *p.Severity != in.Severity {
		if !validSeverity(*p.Severity) {
			return nil, errors.New("severity must be sev1, sev2 or sev3")
		}
		changes = append(changes, "severity "+in.Severity+" to "+*p.Severity)
		in.Severity = *p.Severity
	}
	if p.Status != nil && *p.Status != in.Status {
		if !validStatus(*p.Status) {
			return nil, errors.New("status must be investigating, identified, monitoring or resolved")
		}
		changes = append(changes, "status "+in.Status+" to "+*p.Status)
		in.Status = *p.Status
		if in.Status == "resolved" {
			in.Resolved = now.Unix()
		} else {
			in.Resolved = 0
		}
	}
	if p.Lead != nil && *p.Lead != in.Lead {
		in.Lead = *p.Lead
		changes = append(changes, "lead: "+s.userName(in.Lead))
	}
	if p.Comms != nil && *p.Comms != in.Comms {
		in.Comms = *p.Comms
		changes = append(changes, "comms: "+s.userName(in.Comms))
	}
	if p.Title != nil && strings.TrimSpace(*p.Title) != "" && *p.Title != in.Title {
		in.Title = strings.TrimSpace(*p.Title)
		changes = append(changes, "renamed")
	}
	if p.Summary != nil {
		in.Summary = *p.Summary
	}
	if p.Postmortem != nil {
		in.Postmortem = *p.Postmortem
	}
	wasPublic, statusChanged := in.Public, in.Status != prevStatus
	if p.Public != nil {
		in.Public = *p.Public
	}
	if p.Components != nil {
		in.Components = cleanIDs(*p.Components)
	}
	if err := s.st.put("incident", in.ID, in); err != nil {
		return nil, err
	}
	s.postToStatusPage(in, wasPublic, statusChanged, p.Note, now)
	if len(changes) > 0 || p.Note != "" {
		text := strings.Join(changes, "; ")
		if p.Note != "" {
			text = strings.TrimSpace(text + ": " + p.Note)
		}
		s.st.addEvent("incident", in.ID, "update", actor, text, now)
		kind := "update"
		if in.Status == "resolved" {
			kind = "resolved"
		}
		s.announceIncident(in, kind, text)
	}
	return in, nil
}

func (s *Server) CommentIncident(id, actor, text string) error {
	if _, err := getOf[Incident](s.st, "incident", id); err != nil {
		return err
	}
	if strings.TrimSpace(text) == "" {
		return errors.New("write something")
	}
	s.st.addEvent("incident", id, "comment", actor, text, s.now())
	return nil
}

// postmortemTemplate is a starting point, filled with what the timeline already knows.
func (s *Server) postmortemTemplate(in *Incident) string {
	var b strings.Builder
	fmt.Fprintf(&b, "# Post-mortem: %s\n\n", in.Title)
	fmt.Fprintf(&b, "Severity: %s\nStarted: %s\n", in.Severity, time.Unix(in.Created, 0).UTC().Format(time.RFC1123))
	if in.Resolved > 0 {
		fmt.Fprintf(&b, "Resolved: %s (%s)\n", time.Unix(in.Resolved, 0).UTC().Format(time.RFC1123), time.Duration(in.Resolved-in.Created)*time.Second)
	}
	b.WriteString("\n## Summary\n\n\n## Impact\n\n\n## Root cause\n\n\n## Timeline\n\n")
	for _, e := range s.st.events("incident", in.ID) {
		fmt.Fprintf(&b, "- %s  %s\n", time.Unix(e.Time, 0).UTC().Format("15:04:05"), e.Text)
	}
	b.WriteString("\n## What went well\n\n\n## What went badly\n\n\n## Action items\n\n")
	return b.String()
}