pkg/monitor: wait for Prometheus sidecars to be ready
When fetching alerts from Thanos Query, we should first ensure that the
queriers are connected to both Prometheus sidecars. Otherwise the
returned data could have gaps which would fail the tests verifying that
the Watchdog alert has been always firing.
@@ -15,6 +15,7 @@ import (
"github.com/openshift/origin/pkg/monitor/monitorapi"
prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
prometheustypes "github.com/prometheus/common/model"
+ "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -83,6 +84,34 @@ func FetchEventIntervalsForAllAlerts(ctx context.Context, restConfig *rest.Confi
return nil, err
}
+ // Ensure that Thanos queriers are connected to both Prometheus sidecars
+ // before fetching the alerts. This avoids retrieving partial data
+ // (possibly with gaps) when after an upgrade, one of the Promehteus
+ // sidecars hasn't been reconnected yet to the Thanos queriers.
+ if err = wait.PollImmediateWithContext(ctx, time.Second, 2*time.Minute, func(context.Context) (bool, error) {
+ v, warningsForQuery, err := prometheusClient.Query(ctx, `min(count by(pod) (thanos_store_nodes_grpc_connections{store_type="sidecar"})) == 2`, time.Time{})
+ if err != nil {
+ return false, err
+ }
+
+ if len(warningsForQuery) > 0 {
+ fmt.Printf("#### warnings \n\t%v\n", strings.Join(warningsForQuery, "\n\t"))
+ }
+
+ if v.Type() != prometheustypes.ValVector {
+ return false, fmt.Errorf("expecting a vector type, got %q", v.Type().String())
+ }
+
+ if len(v.(prometheustypes.Vector)) == 0 {
+ fmt.Printf("#### at least one Prometheus sidecar isn't ready\n")
+ return false, nil
+ }
+
+ return true, nil
+ }); err != nil {
+ return nil, fmt.Errorf("Thanos queriers not connected to all Prometheus sidecars: %w", err)
+ }
+
timeRange := prometheusv1.Range{
Start: startTime,
End: time.Now(),
