Add retry to TCP port check
@@ -6,6 +6,7 @@ import (
"net"
"strconv"
"strings"
+ "time"
"github.com/docker/docker/api/types/container"
"github.com/docker/go-connections/nat"
@@ -529,7 +530,7 @@ func (c *Cluster) runServicePortChecks(ctx context.Context) error {
func checkPlaneTCPPortsFromHost(ctx context.Context, host *hosts.Host, portList []string, planeHosts []*hosts.Host, image string, prsMap map[string]v3.PrivateRegistry) error {
var hosts []string
-
+ var portCheckLogs string
for _, host := range planeHosts {
hosts = append(hosts, host.InternalAddress)
}
@@ -551,6 +552,8 @@ func checkPlaneTCPPortsFromHost(ctx context.Context, host *hosts.Host, portList
Type: "json-file",
},
}
+ for retries := 0; retries < 3; retries++ {
+ logrus.Infof("[network] Checking if host [%s] can connect to host(s) [%s] on port(s) [%s], try #%d", host.Address, strings.Join(hosts, " "), strings.Join(portList, " "), retries+1)
if err := docker.DoRemoveContainer(ctx, host.DClient, PortCheckContainer, host.Address); err != nil {
return err
}
@@ -568,12 +571,14 @@ func checkPlaneTCPPortsFromHost(ctx context.Context, host *hosts.Host, portList
return err
}
logrus.Debugf("[network] Length of containerLog is [%d] on host: %s", len(containerLog), host.Address)
- if len(containerLog) > 0 {
- portCheckLogs := strings.Join(strings.Split(strings.TrimSpace(containerLog), "\n"), ", ")
- return fmt.Errorf("[network] Host [%s] is not able to connect to the following ports: [%s]. Please check network policies and firewall rules", host.Address, portCheckLogs)
- }
+ if len(containerLog) == 0 {
return nil
}
+ portCheckLogs = strings.Join(strings.Split(strings.TrimSpace(containerLog), "\n"), ", ")
+ time.Sleep(5 * time.Second)
+ }
+ return fmt.Errorf("[network] Host [%s] is not able to connect to the following ports: [%s]. Please check network policies and firewall rules", host.Address, portCheckLogs)
+}
func getPortBindings(hostAddress string, portList []string) []nat.PortBinding {
portBindingList := []nat.PortBinding{}
