display clear info with each detected alert
We can parse this in sippy with we extract metadata.
@@ -70,13 +70,14 @@ count_over_time(ALERTS{alertstate="firing",severity!="info",alertname!~"Watchdog
labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
violation := fmt.Sprintf("alert %s fired for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
if cause := allowedFiringAlerts.Matches(series); cause != nil {
- debug.Insert(fmt.Sprintf("%s (allowed: %s)", violation, cause.Text))
+ // TODO: this seems to never be firing? no search.ci results show allowed
+ debug.Insert(fmt.Sprintf("%s result=allowed (%s)", violation, cause.Text))
continue
}
if cause := firingAlertsWithBugs.Matches(series); cause != nil {
- knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
+ knownViolations.Insert(fmt.Sprintf("%s result=allowed bug=%s", violation, cause.Text))
} else {
- unexpectedViolations.Insert(violation)
+ unexpectedViolations.Insert(fmt.Sprintf("%s result=failure", violation))
}
}
@@ -98,15 +99,16 @@ sort_desc(
labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
violation := fmt.Sprintf("alert %s pending for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
if cause := allowedPendingAlerts.Matches(series); cause != nil {
- debug.Insert(fmt.Sprintf("%s (allowed: %s)", violation, cause.Text))
+ // TODO: this seems to never be firing? no search.ci results show allowed
+ debug.Insert(fmt.Sprintf("%s result=allowed (%s)", violation, cause.Text))
continue
}
if cause := pendingAlertsWithBugs.Matches(series); cause != nil {
- knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
+ knownViolations.Insert(fmt.Sprintf("%s result=allowed bug=%s", violation, cause.Text))
} else {
// treat pending errors as a flake right now because we are still trying to determine the scope
// TODO: move this to unexpectedViolations later
- unexpectedViolationsAsFlakes.Insert(violation)
+ unexpectedViolationsAsFlakes.Insert(fmt.Sprintf("%s result=allowed", violation))
}
}
