Set Backoff.MaxDelay to 10 seconds
@@ -27,9 +27,11 @@ package rpc
import (
"context"
"crypto/tls"
+ "time"
"github.com/gogo/status"
"google.golang.org/grpc"
+ "google.golang.org/grpc/backoff"
"google.golang.org/grpc/credentials"
"go.temporal.io/server/common/headers"
@@ -38,9 +40,12 @@ import (
const (
// DefaultServiceConfig is a default gRPC connection service config which enables DNS round robin between IPs.
- // To use DNS resolver, a "dns:///" prefix should be applied to the hostPort
+ // To use DNS resolver, a "dns:///" prefix should be applied to the hostPort.
// https://github.com/grpc/grpc/blob/master/doc/naming.md
DefaultServiceConfig = `{"loadBalancingConfig": [{"round_robin":{}}]}`
+
+ // MaxBackoffDelay is a maximum interval between reconnect attempts.
+ MaxBackoffDelay = 10 * time.Second
)
// Dial creates a client connection to the given target with default options.
@@ -54,6 +59,16 @@ func Dial(hostName string, tlsConfig *tls.Config) (*grpc.ClientConn, error) {
grpcSecureOpt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
}
+ // gRPC maintains connection pool inside grpc.ClientConn.
+ // This connection pool has auto reconnect feature.
+ // If connection goes down, gRPC will try to reconnect using exponential backoff strategy:
+ // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md.
+ // Default MaxDelay is 120 seconds which is too high.
+ var cp = grpc.ConnectParams{
+ Backoff: backoff.DefaultConfig,
+ }
+ cp.Backoff.MaxDelay = MaxBackoffDelay
+
return grpc.Dial(hostName,
grpcSecureOpt,
grpc.WithChainUnaryInterceptor(
@@ -61,6 +76,7 @@ func Dial(hostName string, tlsConfig *tls.Config) (*grpc.ClientConn, error) {
errorInterceptor),
grpc.WithDefaultServiceConfig(DefaultServiceConfig),
grpc.WithDisableServiceConfig(),
+ grpc.WithConnectParams(cp),
)
}
