test/extended/etcd: make excessive leader_change test actionable
@@ -2,6 +2,7 @@ package etcd
import (
"context"
+ "fmt"
"time"
g "github.com/onsi/ginkgo"
@@ -21,6 +22,9 @@ var _ = g.Describe("[sig-etcd] etcd", func() {
g.By("Examining the rate of increase in the number of etcd leadership changes for last five minutes")
result, _, err := prometheus.Query(context.Background(), "increase((max by (job) (etcd_server_leader_changes_seen_total) or 0*absent(etcd_server_leader_changes_seen_total))[15m:1m])", time.Now())
o.Expect(err).ToNot(o.HaveOccurred())
- o.Expect(result.(model.Vector)[0].Value).To(o.BeNumerically("==", 0))
+ leaderChangeLastFiveMinutes := result.(model.Vector)[0].Value
+ if leaderChangeLastFiveMinutes != 0 {
+ o.Expect(fmt.Errorf("Leader changes observed last 5m %q: Leader changes are a result of stopping the etcd leader process or from latency (disk or network), review etcd performance metrics", leaderChangeLastFiveMinutes)).ToNot(o.HaveOccurred())
+ }
})
})
