daemon/update: retry pivot 5 times on failed rc
In the spirit of retrying all the things that could fail transiently,
just wrap the `/bin/pivot` call in a retry loop.
Closes:
@@ -12,6 +12,7 @@ import (
"path/filepath"
"reflect"
"strconv"
+ "syscall"
"time"
errors "github.com/pkg/errors"
@@ -22,6 +23,7 @@ import (
"github.com/vincent-petithory/dataurl"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/wait"
)
const (
@@ -599,7 +601,26 @@ func (dn *Daemon) updateOS(oldConfig, newConfig *mcfgv1.MachineConfig) error {
}
glog.Infof("Updating OS to %s", newConfig.Spec.OSImageURL)
- return dn.NodeUpdaterClient.RunPivot(newConfig.Spec.OSImageURL)
+
+ // try to run pivot 5 times; in the future we'd make the tool smarter
+ // so it can always retry things it knows are transient
+ return wait.ExponentialBackoff(wait.Backoff{
+ Steps: 5, // times to retry
+ Duration: 5 * time.Second, // sleep between tries
+ Factor: 2, // factor by which to increase sleep
+ }, func() (bool, error) {
+ var err error
+ if err = dn.NodeUpdaterClient.RunPivot(newConfig.Spec.OSImageURL); err != nil {
+ if exitError, ok := err.(*exec.ExitError); ok {
+ rc := exitError.Sys().(syscall.WaitStatus).ExitStatus()
+ if rc != 0 {
+ glog.Warningf("pivot exited with rc=%d; retrying...", rc)
+ return false, nil
+ }
+ }
+ }
+ return true, err
+ })
}
// Log a message to the systemd journal as well as our stdout
