daemon: make cordon/uncordon more robust
Before marking cordon/uncordon successful,
also check the node.Spec.Unschedulable has been set
correctly.
Also added additional log while performing cordon/uncordon
This is to help debug bugs such as
@@ -39,12 +39,31 @@ func (dn *Daemon) cordonOrUncordonNode(desired bool) error {
}
var lastErr error
if err := wait.ExponentialBackoff(backoff, func() (bool, error) {
+ // Log has been added to ensure that MCO is correctly performing cordon/uncordon.
+ // This should help us with debugging bugs like https://bugzilla.redhat.com/show_bug.cgi?id=2022387
+ glog.Infof("Initiating %s on node (currently schedulable: %t)", verb, !dn.node.Spec.Unschedulable)
err := drain.RunCordonOrUncordon(dn.drainer, dn.node, desired)
if err != nil {
lastErr = err
glog.Infof("%s failed with: %v, retrying", verb, err)
return false, nil
}
+
+ // Re-fetch node so that we are not using cached information
+ var node *corev1.Node
+ if node, err = dn.nodeLister.Get(dn.node.GetName()); err != nil {
+ lastErr = err
+ glog.Errorf("Failed to fetch node %v, retrying", err)
+ return false, nil
+ }
+
+ if node.Spec.Unschedulable != desired {
+ // See https://bugzilla.redhat.com/show_bug.cgi?id=2022387
+ glog.Infof("RunCordonOrUncordon() succeeded but node is still not in %s state, retrying", verb)
+ return false, nil
+ }
+
+ glog.Infof("%s succeeded on node (currently schedulable: %t)", verb, !node.Spec.Unschedulable)
return true, nil
}); err != nil {
if err == wait.ErrWaitTimeout {
