deployment: retry failed watch when observed controller has old resource version
@@ -6,10 +6,12 @@ import (
"time"
kapi "k8s.io/kubernetes/pkg/api"
+ "k8s.io/kubernetes/pkg/api/unversioned"
kcoreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
"k8s.io/kubernetes/pkg/fields"
"k8s.io/kubernetes/pkg/watch"
+ "github.com/golang/glog"
"github.com/openshift/origin/pkg/deploy/api"
deployutil "github.com/openshift/origin/pkg/deploy/util"
)
@@ -17,6 +19,7 @@ import (
var (
// ErrUnknownDeploymentPhase is returned for WaitForRunningDeployment if an unknown phase is returned.
ErrUnknownDeploymentPhase = errors.New("unknown deployment phase")
+ ErrTooOldResourceVersion = errors.New("too old resource version")
)
// WaitForRunningDeployment waits until the specified deployment is no longer New or Pending. Returns true if
@@ -33,6 +36,15 @@ func WaitForRunningDeployment(rn kcoreclient.ReplicationControllersGetter, obser
if _, err := watch.Until(timeout, w, func(e watch.Event) (bool, error) {
if e.Type == watch.Error {
+ // When we send too old resource version in observed replication controller to
+ // watcher, restart the watch with latest available controller.
+ switch t := e.Object.(type) {
+ case *unversioned.Status:
+ if t.Reason == unversioned.StatusReasonGone {
+ glog.V(5).Infof("encountered error while watching for replication controller: %v (retrying)", t)
+ return false, ErrTooOldResourceVersion
+ }
+ }
return false, fmt.Errorf("encountered error while watching for replication controller: %v", e.Object)
}
obj, isController := e.Object.(*kapi.ReplicationController)
@@ -49,6 +61,13 @@ func WaitForRunningDeployment(rn kcoreclient.ReplicationControllersGetter, obser
return false, ErrUnknownDeploymentPhase
}
}); err != nil {
+ if err == ErrTooOldResourceVersion {
+ latestRC, err := rn.ReplicationControllers(observed.Namespace).Get(observed.Name)
+ if err != nil {
+ return observed, false, err
+ }
+ return WaitForRunningDeployment(rn, latestRC, timeout)
+ }
return observed, false, err
}
