service start: add backoff to service restarts
User existing backoff package
Change Info log message to Warn
add variable names to NewBackoff arguments
@@ -17,7 +17,10 @@ import (
"errors"
"fmt"
"io"
+ "math"
+ "time"
+ "github.com/aws/amazon-ecs-init/ecs-init/backoff"
"github.com/aws/amazon-ecs-init/ecs-init/cache"
"github.com/aws/amazon-ecs-init/ecs-init/config"
"github.com/aws/amazon-ecs-init/ecs-init/docker"
@@ -33,6 +36,11 @@ const (
terminalSuccessAgentExitCode = 0
terminalFailureAgentExitCode = 5
upgradeAgentExitCode = 42
+ serviceStartMinRetryTime = time.Millisecond * 500
+ serviceStartMaxRetryTime = time.Second * 15
+ serviceStartRetryJitter = 0.10
+ serviceStartRetryMultiplier = 2.0
+ serviceStartMaxRetries = math.MaxInt64 // essentially retry forever
)
// Engine contains methods invoked when ecs-init is run
@@ -168,7 +176,9 @@ func (e *Engine) load(image io.ReadCloser, err error) error {
// StartSupervised starts the ECS Agent and ensures it stays running, except for terminal errors (indicated by an agent exit code of 5)
func (e *Engine) StartSupervised() error {
agentExitCode := -1
- for agentExitCode != terminalSuccessAgentExitCode && agentExitCode != terminalFailureAgentExitCode {
+ retryBackoff := backoff.NewBackoff(serviceStartMinRetryTime, serviceStartMaxRetryTime,
+ serviceStartRetryJitter, serviceStartRetryMultiplier, serviceStartMaxRetries)
+ for {
err := e.docker.RemoveExistingAgentContainer()
if err != nil {
return engineError("could not remove existing Agent container", err)
@@ -180,15 +190,24 @@ func (e *Engine) StartSupervised() error {
return engineError("could not start Agent", err)
}
log.Infof("Agent exited with code %d", agentExitCode)
- if agentExitCode == upgradeAgentExitCode {
+
+ switch agentExitCode {
+ case upgradeAgentExitCode:
err = e.upgradeAgent()
if err != nil {
log.Error("could not upgrade agent", err)
+ } else {
+ // continuing here because a successful upgrade doesn't need to backoff retries
+ continue
}
- }
- }
- if agentExitCode == terminalFailureAgentExitCode {
+ case terminalFailureAgentExitCode:
return errors.New("agent exited with terminal exit code")
+ case terminalSuccessAgentExitCode:
+ return nil
+ }
+ d := retryBackoff.Duration()
+ log.Warnf("ECS Agent failed to start, retrying in %s", d)
+ time.Sleep(d)
}
return nil
}
