extend tcp dial timeout and http response header timeout
@@ -284,6 +284,8 @@ func (b *BackendSampler) wrapWithAuth(rt http.RoundTripper) (http.RoundTripper,
func (b *BackendSampler) GetHTTPClient() (*http.Client, error) {
timeoutForEntireRequest := b.getTimeout()
timeoutForPartOfRequest := timeoutForEntireRequest / 5 // this is less so that we can see failures for individual portions of the request
+ tcpDialTimeout := timeoutForPartOfRequest * 2 // we see more tcp i/o timeouts than expected at 1s
+ responseHeaderTimeout := timeoutForPartOfRequest * 2 // we see more timeouts waiting for response headers than expected at 1s
b.initHTTPClient.Do(func() {
var httpTransport *http.Transport
@@ -291,26 +293,26 @@ func (b *BackendSampler) GetHTTPClient() (*http.Client, error) {
case NewConnectionType:
httpTransport = &http.Transport{
Dial: (&net.Dialer{
- Timeout: timeoutForPartOfRequest,
+ Timeout: tcpDialTimeout,
KeepAlive: -1, // this looks unnecessary to me, but it was set in other code.
}).Dial,
TLSClientConfig: b.getTLSConfig(),
DisableKeepAlives: true, // this prevents connections from being reused
TLSHandshakeTimeout: timeoutForPartOfRequest,
IdleConnTimeout: timeoutForPartOfRequest,
- ResponseHeaderTimeout: timeoutForPartOfRequest,
+ ResponseHeaderTimeout: responseHeaderTimeout,
ExpectContinueTimeout: timeoutForPartOfRequest,
}
case ReusedConnectionType:
httpTransport = &http.Transport{
Dial: (&net.Dialer{
- Timeout: b.getTimeout(),
+ Timeout: tcpDialTimeout,
}).Dial,
TLSClientConfig: b.getTLSConfig(),
TLSHandshakeTimeout: timeoutForPartOfRequest,
IdleConnTimeout: timeoutForPartOfRequest,
- ResponseHeaderTimeout: timeoutForPartOfRequest,
+ ResponseHeaderTimeout: responseHeaderTimeout,
ExpectContinueTimeout: timeoutForPartOfRequest,
}
@@ -407,7 +409,7 @@ func (b *BackendSampler) RunEndpointMonitoring(ctx context.Context, monitorRecor
defer consumerCancel()
go func() {
<-producerContext.Done()
- consumptionGrace := 15 * time.Second
+ consumptionGrace := b.getTimeout() * 3 // we need to wait longer than backstopContextTimeout to ensure we're finished producing
time.Sleep(consumptionGrace)
consumerCancel()
}()
