test/monitoring: add retry logic to control plane metrics test
As the metrics may not be immidiately available.
@@ -88,13 +88,21 @@ func testComponentsPrometheusMetrics(t *testing.T, v1api v1.API) {
const contextTimeout = 10
+ var err error
+
+ // This loop ensures that we try to query prometheus for the state of targets multiple times.
+ // This is the retry logic.
+ for i := 0; i < 20; i++ {
+ t.Logf("Running scrape test iteration #%d", i)
+
+ // Use function to be able to use defer.
+ err = func() error {
ctx, cancel := context.WithTimeout(context.Background(), contextTimeout*time.Second)
defer cancel()
results, warnings, err := v1api.Query(ctx, tc.query, time.Now())
if err != nil {
- t.Errorf("error querying Prometheus: %v", err)
- return
+ return fmt.Errorf("error querying Prometheus: %w", err)
}
if len(warnings) > 0 {
@@ -102,10 +110,27 @@ func testComponentsPrometheusMetrics(t *testing.T, v1api v1.API) {
}
if len(results.String()) == 0 {
- t.Fatalf("no metrics found")
+ return fmt.Errorf("no metrics found")
}
t.Logf("found %d results for %s", len(strings.Split(results.String(), "\n")), tc.query)
+
+ return nil
+ }()
+
+ // If there is no errors, break the retry loop.
+ if err == nil {
+ break
+ }
+
+ // Wait a bit before next attempt.
+ time.Sleep(30 * time.Second) //nolint:gomnd
+ }
+
+ // If there are still some errors after all retries, fail the test.
+ if err != nil {
+ t.Fatal(err)
+ }
})
}
}
