Compare the most recent observation to the second most recent observation and clear out old entires if we detect that the pod restart count has gone down, aka the pod has been restarted but came back with the same name.
@@ -159,6 +159,8 @@ func (prc *Checker) doChecks() error {
// ReapPodRestartChecks reaps old data from PodRestartCheck samplings
func (prc *Checker) reapPodRestartChecks(currentPods *v1.PodList) {
+ var mostRecentObservation RestartCountObservation
+ var secondMostRecentObservation RestartCountObservation
for podName, restartObservations := range prc.RestartObservations {
// if the pod no longer exists, then delete its observations
if !podInPodList(podName, currentPods) {
@@ -171,6 +173,30 @@ func (prc *Checker) reapPodRestartChecks(currentPods *v1.PodList) {
if observation.Time.Before(time.Now().Add(-time.Hour)) {
prc.RestartObservations[podName] = deleteFromSlice(i, prc.RestartObservations[podName])
}
+ // Find the most recent PRC observation
+ if observation.Time.After(mostRecentObservation.Time) {
+ mostRecentObservation = observation
+ }
+ // Find the second most recent PRC observation
+ if observation.Time != time.isZero() && observation.Time != mostRecentObservation.Time && observation.Time.After(secondMostRecentObservation.Time) {
+ secondMostRecentObservation = observation
+ }
+ }
+ }
+ // If the most recent observation restart count is less than the previous observation restart count and is now 0, delete all the
+ // entries that arent the most recent one, assuming that the pod was restarted but still has the same name. This is a new
+ // pod with a reset restart counter but has the same exact name as the previous pod that existed but was deleted
+ // we see this in stateful sets
+ if mostRecentObservation.Count < secondMostRecentObservation.Count {
+ for podName, restartObservations := range prc.RestartObservations {
+ for i, observation := range restartObservations{
+ if observation != mostRecentObservation {
+ // delete all the old entries
+ prc.RestartObservations[podName] = deleteFromSlice(i, prc.RestartObservations[podName])
+ // re-add the most recent one
+ prc.RestartObservations[podName] = append(prc.RestartObservations[PodName], observation)
+ }
+ }
}
}
}
