Retry kubeconfig checks, when kube-apiserver is temporarily unavailable
@@ -3,12 +3,15 @@ package apiserver
import (
"context"
"fmt"
+ "regexp"
g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"
- exutil "github.com/openshift/origin/test/extended/util"
+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
+
+ exutil "github.com/openshift/origin/test/extended/util"
)
var _ = g.Describe("[Conformance][sig-api-machinery][Feature:APIServer] local kubeconfig", func() {
@@ -29,13 +32,29 @@ var _ = g.Describe("[Conformance][sig-api-machinery][Feature:APIServer] local ku
framework.Logf("Discovered %d master nodes.", len(masterNodes.Items))
o.Expect(masterNodes.Items).NotTo(o.HaveLen(0))
for _, master := range masterNodes.Items {
- g.By("Testing master node " + master.Name)
- kubeconfigPath := "/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/node-kubeconfigs/" + kubeconfig
- framework.Logf("Verifying kubeconfig %q on master %s", master.Name)
- out, err := oc.AsAdmin().Run("debug").Args("node/"+master.Name, "--", "chroot", "/host", "/bin/bash", "-euxo", "pipefail", "-c", fmt.Sprintf(`oc --kubeconfig "%s" get namespace kube-system`, kubeconfigPath)).Output()
+ retry, err := testNode(oc, kubeconfig, master.Name)
+ for retries := 2; retries > 0; retries-- {
+ if !retry {
+ break
+ }
+ g.By("There was a retryable error for " + fmt.Sprintf("%s/%s", master.Name, kubeconfig))
+ retry, err = testNode(oc, kubeconfig, master.Name)
+ }
o.Expect(err).NotTo(o.HaveOccurred())
- framework.Logf(out)
}
})
}
})
+
+func testNode(oc *exutil.CLI, kubeconfig, masterName string) (bool, error) {
+ g.By("Testing master node " + masterName)
+ kubeconfigPath := "/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/node-kubeconfigs/" + kubeconfig
+ framework.Logf("Verifying kubeconfig %q on master %q", kubeconfig, masterName)
+ out, err := oc.AsAdmin().Run("debug").Args("node/"+masterName, "--", "chroot", "/host", "/bin/bash", "-euxo", "pipefail", "-c",
+ fmt.Sprintf(`oc --kubeconfig "%s" get namespace kube-system`, kubeconfigPath)).Output()
+ framework.Logf(out)
+ // retry error when kube-apiserver was temporarily unavailable, this matches oc error coming from:
+ // https://github.com/kubernetes/kubernetes/blob/cbb5ea8210596ada1efce7e7a271ca4217ae598e/staging/src/k8s.io/kubectl/pkg/cmd/util/helpers.go#L237-L243
+ matched, _ := regexp.MatchString("The connection to the server .+ was refused - did you specify the right host or port", out)
+ return !matched, err
+}
