Merge pull request coroot#621 from coroot/incidents_reworked

Incident data materialization in database

Author: def

api/api.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"slices"
 	"sort"
+	"strconv"
 	"time"
 
 	"github.com/coroot/coroot/api/forms"
@@ -364,7 +365,7 @@ func (api *Api) Overview(w http.ResponseWriter, r *http.Request, u *db.User) {
 		return
 	}
 	var ch *clickhouse.Client
-	if ch, err = api.getClickhouseClient(project); err != nil {
+	if ch, err = api.GetClickhouseClient(project); err != nil {
 		klog.Warningln(err)
 	}
 	auditor.Audit(world, project, nil, project.ClickHouseConfig(api.globalClickHouse) != nil, nil)
@@ -929,6 +930,49 @@ func (api *Api) RCA(w http.ResponseWriter, r *http.Request, u *db.User) {
 	utils.WriteJson(w, api.WithContext(project, cacheStatus, world, "not implemented"))
 }
 
+func (api *Api) Incidents(w http.ResponseWriter, r *http.Request, u *db.User) {
+	vars := mux.Vars(r)
+	projectId := db.ProjectId(vars["project"])
+	limit := 100
+	if l := r.URL.Query().Get("limit"); l != "" {
+		l64, err := strconv.ParseUint(l, 10, 32)
+		if err != nil {
+			klog.Warningln("invalid limit:", l)
+			http.Error(w, "", http.StatusBadRequest)
+			return
+		}
+		limit = int(l64)
+	}
+	project, err := api.db.GetProject(projectId)
+	if err != nil {
+		if errors.Is(err, db.ErrNotFound) {
+			http.Error(w, "project not found", http.StatusNotFound)
+			klog.Warningln("project not found:", projectId)
+			return
+		}
+		klog.Errorln(err)
+		http.Error(w, "", http.StatusInternalServerError)
+		return
+	}
+	incidents, err := api.db.GetLatestIncidents(project.Id, limit)
+	if err != nil {
+		klog.Errorln(err)
+		http.Error(w, "", http.StatusInternalServerError)
+		return
+	}
+	world, project, cacheStatus, err := api.LoadWorldByRequest(r)
+	if err != nil {
+		klog.Errorln(err)
+		http.Error(w, "", http.StatusInternalServerError)
+		return
+	}
+	if project == nil || world == nil {
+		utils.WriteJson(w, api.WithContext(project, cacheStatus, world, nil))
+		return
+	}
+	utils.WriteJson(w, api.WithContext(project, cacheStatus, world, views.Incidents(world, incidents)))
+}
+
 func (api *Api) Incident(w http.ResponseWriter, r *http.Request, u *db.User) {
 	vars := mux.Vars(r)
 	projectId := vars["project"]
@@ -1239,7 +1283,7 @@ func (api *Api) Profiling(w http.ResponseWriter, r *http.Request, u *db.User) {
 		return
 	}
 	var ch *clickhouse.Client
-	if ch, err = api.getClickhouseClient(project); err != nil {
+	if ch, err = api.GetClickhouseClient(project); err != nil {
 		klog.Warningln(err)
 		http.Error(w, "ClickHouse is not available", http.StatusInternalServerError)
 		return
@@ -1296,7 +1340,7 @@ func (api *Api) Tracing(w http.ResponseWriter, r *http.Request, u *db.User) {
 	}
 	q := r.URL.Query()
 	var ch *clickhouse.Client
-	if ch, err = api.getClickhouseClient(project); err != nil {
+	if ch, err = api.GetClickhouseClient(project); err != nil {
 		klog.Warningln(err)
 		http.Error(w, "ClickHouse is not available", http.StatusInternalServerError)
 		return
@@ -1350,7 +1394,7 @@ func (api *Api) Logs(w http.ResponseWriter, r *http.Request, u *db.User) {
 		http.Error(w, "Application not found", http.StatusNotFound)
 		return
 	}
-	ch, chErr := api.getClickhouseClient(project)
+	ch, chErr := api.GetClickhouseClient(project)
 	if chErr != nil {
 		klog.Warningln(chErr)
 	}
@@ -1535,12 +1579,11 @@ func (api *Api) getTimeContext(r *http.Request) (from timeseries.Time, to timese
 		if incident, err := api.db.GetIncidentByKey(projectId, incidentKey); err != nil {
 			klog.Warningln("failed to get incident:", err)
 		} else {
-			margin := model.MaxAlertRuleShortWindow + 15*timeseries.Minute
-			from = incident.OpenedAt.Add(-margin)
+			from = incident.OpenedAt.Add(-model.IncidentTimeOffset)
 			if incident.Resolved() {
-				if t := incident.ResolvedAt.Add(margin); t.Before(to) {
-					to = t
-				}
+				to = incident.ResolvedAt.Add(model.IncidentTimeOffset)
+			} else {
+				to = now
 			}
 		}
 	}
@@ -1571,7 +1614,7 @@ func maxDuration(d1, d2 timeseries.Duration) timeseries.Duration {
 	return d2
 }
 
-func (api *Api) getClickhouseClient(project *db.Project) (*clickhouse.Client, error) {
+func (api *Api) GetClickhouseClient(project *db.Project) (*clickhouse.Client, error) {
 	cfg := project.ClickHouseConfig(api.globalClickHouse)
 	if cfg == nil {
 		return nil, nil

api/ctx.go

@@ -15,8 +15,9 @@ type DataWithContext struct {
 }
 
 type Context struct {
-	Status Status `json:"status"`
-	Search Search `json:"search"`
+	Status    Status                            `json:"status"`
+	Search    Search                            `json:"search"`
+	Incidents map[model.ApplicationCategory]int `json:"incidents"`
 }
 
 type Status struct {
@@ -62,13 +63,27 @@ type Node struct {
 func (api *Api) WithContext(p *db.Project, cacheStatus *cache.Status, w *model.World, data any) DataWithContext {
 	return DataWithContext{
 		Context: Context{
-			Status: renderStatus(p, cacheStatus, w, api.globalPrometheus),
-			Search: renderSearch(w),
+			Status:    renderStatus(p, cacheStatus, w, api.globalPrometheus),
+			Search:    renderSearch(w),
+			Incidents: renderIncidents(w),
 		},
 		Data: data,
 	}
 }
 
+func renderIncidents(w *model.World) map[model.ApplicationCategory]int {
+	res := map[model.ApplicationCategory]int{}
+	for _, app := range w.Applications {
+		if len(app.Incidents) == 0 {
+			continue
+		}
+		if last := app.Incidents[len(app.Incidents)-1]; !last.Resolved() {
+			res[app.Category]++
+		}
+	}
+	return res
+}
+
 func renderStatus(p *db.Project, cacheStatus *cache.Status, w *model.World, globalPrometheus *db.IntegrationPrometheus) Status {
 	res := Status{
 		Status: model.OK,

api/forms/forms.go

@@ -670,11 +670,11 @@ func testIncidentNotification(project *db.Project) *db.IncidentNotification {
 		ProjectId:     project.Id,
 		ApplicationId: model.NewApplicationId("default", model.ApplicationKindDeployment, "fake-app"),
 		IncidentKey:   "123ab456",
-		Status:        model.WARNING,
+		Status:        model.CRITICAL,
 		Details: &db.IncidentNotificationDetails{
 			Reports: []db.IncidentNotificationDetailsReport{
-				{Name: model.AuditReportSLO, Check: model.Checks.SLOLatency.Title, Message: "error budget burn rate is 20x within 1 hour"},
 				{Name: model.AuditReportNetwork, Check: model.Checks.NetworkRTT.Title, Message: "high network latency to 2 upstream services"},
+				{Name: model.AuditReportLogs, Check: model.Checks.LogErrors.Title, Message: "1206 errors occurred"},
 			},
 		},
 	}

api/views/incident/incident.go

@@ -2,36 +2,133 @@ package incident
 
 import (
 	"github.com/coroot/coroot/model"
+	"github.com/coroot/coroot/timeseries"
+	"github.com/coroot/coroot/utils"
 )
 
+type Incident struct {
+	model.ApplicationIncident
+	Impact              float32                   `json:"impact"`
+	ShortDescription    string                    `json:"short_description"`
+	ApplicationCategory model.ApplicationCategory `json:"application_category"`
+	Duration            timeseries.Duration       `json:"duration"`
+}
+
+func RenderList(w *model.World, incidents []*model.ApplicationIncident) []Incident {
+	res := make([]Incident, 0, len(incidents))
+
+	for _, i := range incidents {
+		res = append(res, renderIncident(w, i))
+	}
+	return res
+}
+
+func renderIncident(w *model.World, i *model.ApplicationIncident) Incident {
+	category := model.ApplicationCategoryApplication
+	if app := w.GetApplication(i.ApplicationId); app != nil {
+		category = app.Category
+	}
+	to := timeseries.Now()
+	if i.Resolved() {
+		to = i.ResolvedAt
+	}
+	return Incident{
+		ApplicationIncident: *i,
+		ShortDescription:    i.ShortDescription(),
+		ApplicationCategory: category,
+		Duration:            to.Sub(i.OpenedAt),
+		Impact: max(
+			i.Details.LatencyImpact.AffectedRequestPercentage,
+			i.Details.AvailabilityImpact.AffectedRequestPercentage,
+		),
+	}
+}
+
+type SLODetails struct {
+	Objective  string  `json:"objective"`
+	Compliance string  `json:"compliance"`
+	Violated   bool    `json:"violated"`
+	Threshold  float32 `json:"threshold"`
+}
+
 type View struct {
-	Summary
-	HeatMap *model.Widget `json:"heatmap"`
+	Incident
+	AvailabilitySLO *SLODetails     `json:"availability_slo,omitempty"`
+	LatencySLO      *SLODetails     `json:"latency_slo,omitempty"`
+	ActualFrom      timeseries.Time `json:"actual_from"`
+	ActualTo        timeseries.Time `json:"actual_to"`
+
+	Widgets []*model.Widget `json:"widgets"`
 }
 
 func Render(w *model.World, app *model.Application, incident *model.ApplicationIncident) *View {
-	v := &View{Summary: CalcSummary(w, app, incident), HeatMap: getHeatMap(app)}
-	if v.HeatMap == nil {
-		return nil
+	to := timeseries.Now()
+	if incident.Resolved() {
+		to = incident.ResolvedAt
+	}
+	v := &View{
+		Incident: renderIncident(w, incident),
+		ActualTo: to,
+		Widgets:  incidentWidgets(w, app),
+	}
+	if len(app.AvailabilitySLIs) > 0 {
+		sli := app.AvailabilitySLIs[0]
+		v.AvailabilitySLO = &SLODetails{
+			Objective:  utils.FormatPercentage(sli.Config.ObjectivePercentage) + " of requests should not fail",
+			Compliance: "100%",
+		}
+		for _, br := range incident.Details.AvailabilityBurnRates {
+			if br.Severity > model.OK {
+				if t := incident.OpenedAt.Add(-br.ShortWindow); v.ActualFrom.IsZero() || t.After(v.ActualFrom) {
+					v.ActualFrom = t
+				}
+				v.AvailabilitySLO.Violated = true
+				v.AvailabilitySLO.Compliance = utils.FormatPercentage(100 - br.LongWindowPercentage)
+				break
+			}
+		}
 	}
-	v.HeatMap.AddAnnotation(model.Annotation{Name: "incident", X1: v.ActualFrom, X2: v.ActualTo})
-	return v
-}
 
-func getHeatMap(app *model.Application) *model.Widget {
-	var sloReport *model.AuditReport
-	for _, r := range app.Reports {
-		if r.Name == model.AuditReportSLO {
-			sloReport = r
+	if len(app.LatencySLIs) > 0 {
+		sli := app.LatencySLIs[0]
+		v.LatencySLO = &SLODetails{
+			Objective:  utils.FormatPercentage(sli.Config.ObjectivePercentage) + " of requests should be served faster than " + utils.FormatLatency(sli.Config.ObjectiveBucket),
+			Compliance: "100%",
+			Threshold:  sli.Config.ObjectiveBucket,
+		}
+		for _, br := range incident.Details.LatencyBurnRates {
+			if v.ActualFrom.IsZero() {
+				v.ActualFrom = incident.OpenedAt.Add(-br.ShortWindow)
+			}
+			if br.Severity > model.OK {
+				if t := incident.OpenedAt.Add(-br.ShortWindow); v.ActualFrom.IsZero() || t.After(v.ActualFrom) {
+					v.ActualFrom = t
+				}
+				v.LatencySLO.Violated = true
+				v.LatencySLO.Compliance = utils.FormatPercentage(100 - br.LongWindowPercentage)
+				break
+			}
 		}
 	}
-	if sloReport == nil {
-		return nil
+	for _, widget := range v.Widgets {
+		widget.AddAnnotation(model.Annotation{Name: "incident", X1: v.ActualFrom, X2: to})
 	}
-	for _, w := range sloReport.Widgets {
-		if w.Heatmap != nil {
-			return w
-		}
+	return v
+}
+
+func incidentWidgets(w *model.World, app *model.Application) []*model.Widget {
+	var res []*model.Widget
+	if len(app.LatencySLIs) > 0 {
+		ch := model.NewChart(w.Ctx, "Latency, seconds").
+			PercentilesFrom(app.LatencySLIs[0].Histogram, 0.25, 0.5, 0.75, 0.95, 0.99)
+		res = append(res, &model.Widget{Chart: ch})
+	}
+	if len(app.AvailabilitySLIs) > 0 {
+		res = append(res, &model.Widget{
+			Chart: model.NewChart(w.Ctx, "Errors, per second").
+				AddSeries("errors", app.AvailabilitySLIs[0].FailedRequests.Map(timeseries.NanToZero), "black").
+				Stacked(),
+		})
 	}
-	return nil
+	return res
 }