controller: Improve log for the unknown replica detection and cleanup
@@ -307,6 +307,7 @@ func (ec *EngineController) syncEngine(key string) (err error) {
syncReplicaAddressMap = true
}
if syncReplicaAddressMap && !reflect.DeepEqual(engine.Status.CurrentReplicaAddressMap, engine.Spec.ReplicaAddressMap) {
+ log.Debug("Updating engine current replica address map")
engine.Status.CurrentReplicaAddressMap = engine.Spec.ReplicaAddressMap
// Make sure the CurrentReplicaAddressMap persist in the etcd before continue
return nil
@@ -775,6 +776,7 @@ func (m *EngineMonitor) refresh(engine *longhorn.Engine) error {
// ReconcileEngineState.
// https://github.com/longhorn/longhorn/issues/4120
currentReplicaModeMap[replica] = r.Mode
+ m.logger.Warnf("Found unknown replica %v in the Replica URL Mode Map", addr)
continue
}
@@ -1336,30 +1338,31 @@ func GetBinaryClientForEngine(e *longhorn.Engine, engines engineapi.EngineClient
}
func (ec *EngineController) removeUnknownReplica(e *longhorn.Engine) error {
- unknownReplicaURLs := []string{}
- for replica := range e.Status.ReplicaModeMap {
+ unknownReplicaMap := map[string]longhorn.ReplicaMode{}
+ for replica, mode := range e.Status.ReplicaModeMap {
// unknown replicas have been named as `unknownReplicaPrefix-<replica URL>`
if strings.HasPrefix(replica, unknownReplicaPrefix) {
- unknownReplicaURLs = append(unknownReplicaURLs, strings.TrimPrefix(replica, unknownReplicaPrefix))
+ unknownReplicaMap[strings.TrimPrefix(replica, unknownReplicaPrefix)] = mode
}
}
- if len(unknownReplicaURLs) == 0 {
+ if len(unknownReplicaMap) == 0 {
return nil
}
- for _, url := range unknownReplicaURLs {
+ for url := range unknownReplicaMap {
engineClientProxy, err := ec.getEngineClientProxy(e, e.Status.CurrentImage)
if err != nil {
- return errors.Wrapf(err, "failed to remove unknown replica %v from engine", url)
+ return errors.Wrapf(err, "failed to get the engine client %v when removing unknown replica %v in mode %v from engine", e.Name, url, unknownReplicaMap[url])
}
go func(url string) {
defer engineClientProxy.Close()
+ ec.eventRecorder.Eventf(e, v1.EventTypeNormal, EventReasonDelete, "Removing unknown replica %v in mode %v from engine", url, unknownReplicaMap[url])
if err := engineClientProxy.ReplicaRemove(e, url); err != nil {
- ec.eventRecorder.Eventf(e, v1.EventTypeWarning, EventReasonFailedDeleting, "Failed to remove unknown replica %v from engine: %v", url, err)
+ ec.eventRecorder.Eventf(e, v1.EventTypeWarning, EventReasonFailedDeleting, "Failed to remove unknown replica %v in mode %v from engine: %v", url, unknownReplicaMap[url], err)
} else {
- ec.eventRecorder.Eventf(e, v1.EventTypeNormal, EventReasonDelete, "Removed unknown replica %v from engine", url)
+ ec.eventRecorder.Eventf(e, v1.EventTypeNormal, EventReasonDelete, "Removed unknown replica %v in mode %v from engine", url, unknownReplicaMap[url])
}
}(url)
}
