Add retries in reconcile when delete hosts
@@ -3,6 +3,7 @@ package cluster
import (
"context"
"fmt"
+ "time"
"github.com/rancher/rke/docker"
"github.com/rancher/rke/hosts"
@@ -122,8 +123,20 @@ func reconcileControl(ctx context.Context, currentCluster, kubeCluster *Cluster,
}
func reconcileHost(ctx context.Context, toDeleteHost *hosts.Host, worker, etcd bool, cleanerImage string, dialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, clusterPrefixPath string, clusterVersion string) error {
- if err := toDeleteHost.TunnelUp(ctx, dialerFactory, clusterPrefixPath, clusterVersion); err != nil {
- return fmt.Errorf("Not able to reach the host: %v", err)
+ var retryErr error
+ retries := 3
+ sleepSeconds := 3
+ for i := 0; i < retries; i++ {
+ if retryErr = toDeleteHost.TunnelUp(ctx, dialerFactory, clusterPrefixPath, clusterVersion); retryErr != nil {
+ logrus.Debugf("Failed to dial the host %s trying again in %d seconds", toDeleteHost.Address, sleepSeconds)
+ time.Sleep(time.Second * time.Duration(sleepSeconds))
+ toDeleteHost.DClient = nil
+ continue
+ }
+ break
+ }
+ if retryErr != nil {
+ return fmt.Errorf("Not able to reach the host: %v", retryErr)
}
if worker {
if err := services.RemoveWorkerPlane(ctx, []*hosts.Host{toDeleteHost}, false); err != nil {
