retry Update of condition if needed
With the current code, it can happen that condition Update fails
due to a collision with someone else editing NetworkAddonsConfig
object.
With this patch, we retry the Update several times until it's
successfully updated or out of tries.
@@ -18,6 +18,11 @@ import (
opv1alpha1 "github.com/kubevirt/cluster-network-addons-operator/pkg/apis/networkaddonsoperator/v1alpha1"
)
+const (
+ conditionsUpdateRetries = 10
+ conditionsUpdateCoolDown = 50 * time.Millisecond
+)
+
// StatusLevel is used to sort priority of reported failure conditions. When operator is failing
// on two levels (e.g. handling configuration and deploying pods), only the higher level will be
// reported. This is needed, so failing pods from previous run won't silence failing render of
@@ -45,14 +50,32 @@ func New(client client.Client, name string) *StatusManager {
return &StatusManager{client: client, name: name}
}
-// Set updates the NetworkAddonsConfig.Status with the provided conditions
+// Set updates the NetworkAddonsConfig.Status with the provided conditions.
+// Since Update call can fail due to a collision with someone else writing into
+// the status, calling set is tried several times.
+// TODO: Calling of Patch instead may save some problems. We can reiterate later,
+// current collision problem is detected by functional tests
func (status *StatusManager) Set(conditions ...opv1alpha1.NetworkAddonsCondition) {
+ for i := 0; i < conditionsUpdateRetries; i++ {
+ err := status.set(conditions...)
+ if err == nil {
+ log.Print("Successfully updated status conditions")
+ return
+ }
+ log.Printf("Failed calling status Set %d/%d: %v", i+1, conditionsUpdateRetries, err)
+ time.Sleep(conditionsUpdateCoolDown)
+ }
+ log.Print("Failed to update conditions within given number of retries")
+}
+
+// set updates the NetworkAddonsConfig.Status with the provided conditions
+func (status *StatusManager) set(conditions ...opv1alpha1.NetworkAddonsCondition) error {
// Read the current NetworkAddonsConfig
config := &opv1alpha1.NetworkAddonsConfig{ObjectMeta: metav1.ObjectMeta{Name: status.name}}
err := status.client.Get(context.TODO(), types.NamespacedName{Name: status.name}, config)
if err != nil {
log.Printf("Failed to get NetworkAddonsOperator %q in order to update its State: %v", status.name, err)
- return
+ return nil
}
oldStatus := config.Status.DeepCopy()
@@ -78,14 +101,16 @@ func (status *StatusManager) Set(conditions ...opv1alpha1.NetworkAddonsCondition
}
if reflect.DeepEqual(oldStatus, config.Status) {
- return
+ return nil
}
// Update NetworkAddonsConfig with updated Status field
err = status.client.Status().Update(context.TODO(), config)
if err != nil {
- log.Printf("Failed to update NetworkAddonsConfig %q Status: %v", config.Name, err)
+ return fmt.Errorf("Failed to update NetworkAddonsConfig %q Status: %v", config.Name, err)
}
+
+ return nil
}
// syncFailing syncs the current Failing status
@@ -134,7 +159,6 @@ func (status *StatusManager) SetDeployments(deployments []types.NamespacedName)
status.deployments = deployments
}
-// TODO refactoring
// SetFromPods sets the operator status to Failing, Progressing, or Available, based on
// the current status of the manager's DaemonSets and Deployments. However, this is a
// no-op if the StatusManager is currently marked as failing due to a configuration error.
