controller: skip replica replenishment during detaching
Otherwise, when there are not enough nodes for a attached volume,
the volume controller will blindly create new replicas with
scheduling failure then immediately fail & clean up those replicas
during detachment.
Longhorn 2460
@@ -1587,6 +1587,10 @@ func (vc *VolumeController) replenishReplicas(v *longhorn.Volume, e *longhorn.En
return nil
}
+ if (len(rs) != 0) && v.Status.State != types.VolumeStateAttached {
+ return nil
+ }
+
if e == nil {
return fmt.Errorf("BUG: replenishReplica needs a valid engine")
}
@@ -1638,7 +1642,6 @@ func (vc *VolumeController) replenishReplicas(v *longhorn.Volume, e *longhorn.En
reusableFailedReplica.Name, vc.backoff.Get(reusableFailedReplica.Name).Seconds())
}
if vc.scheduler.RequireNewReplica(rs, v, hardNodeAffinity) {
- log.Debugf("A new replica will be replenished during rebuilding")
if err := vc.createReplica(v, e, rs, hardNodeAffinity, !newVolume); err != nil {
return err
}
@@ -2688,6 +2691,8 @@ func (vc *VolumeController) createEngine(v *longhorn.Volume) (*longhorn.Engine,
}
func (vc *VolumeController) createReplica(v *longhorn.Volume, e *longhorn.Engine, rs map[string]*longhorn.Replica, hardNodeAffinity string, isRebuildingReplica bool) error {
+ log := getLoggerForVolume(vc.logger, v)
+
replica := &longhorn.Replica{
ObjectMeta: metav1.ObjectMeta{
Name: types.GenerateReplicaNameForVolume(v.Name),
@@ -2708,6 +2713,7 @@ func (vc *VolumeController) createReplica(v *longhorn.Volume, e *longhorn.Engine
},
}
if isRebuildingReplica {
+ log.Debugf("A new replica %v will be replenished during rebuilding", replica.Name)
replica.Spec.RebuildRetryCount++
}
