make sure container terminated state is not nil before getting container terminated time
@@ -190,9 +190,11 @@ func (k *KubernetesAPI) deleteFilteredCheckerPods(ctx context.Context, client *k
for n, v := range reapCheckerPods {
+ podTerminated, podTerminatedTime := getPodCompletedTime(v)
+ if podTerminated {
+
// Delete pods older than maxCheckPodAge and is in status Succeeded
- if v.Status.Phase == v1.PodSucceeded {
- if time.Now().Sub(getPodCompletedTime(v)) > cfg.MaxCheckPodAge {
+ if v.Status.Phase == v1.PodSucceeded && time.Now().Sub(podTerminatedTime) > cfg.MaxCheckPodAge {
log.Infoln("checkReaper: Found completed pod older than:", cfg.MaxCheckPodAge, "in status `Succeeded`. Deleting pod:", n)
err = k.deletePod(ctx, v)
@@ -202,11 +204,10 @@ func (k *KubernetesAPI) deleteFilteredCheckerPods(ctx context.Context, client *k
}
delete(reapCheckerPods, n)
}
- }
// Delete failed pods (status Failed) older than maxCheckPodAge
if v.Status.Phase == v1.PodFailed {
- if time.Now().Sub(getPodCompletedTime(v)) > cfg.MaxCheckPodAge {
+ if time.Now().Sub(podTerminatedTime) > cfg.MaxCheckPodAge {
log.Infoln("checkReaper: Found completed pod older than:", cfg.MaxCheckPodAge, "in status `Failed`. Deleting pod:", n)
err = k.deletePod(ctx, v)
@@ -217,6 +218,7 @@ func (k *KubernetesAPI) deleteFilteredCheckerPods(ctx context.Context, client *k
delete(reapCheckerPods, n)
}
}
+ }
// Delete if there are more than MaxCompletedPodCount checker pods with the same name in status Succeeded that were created more recently
// Delete if the checker pod is Failed and there are more than MaxErrorPodCount checker pods of the same type which were created more recently
@@ -279,11 +281,14 @@ func getAllCompletedPodsWithCheckName(reapCheckerPods map[string]v1.Pod, pod v1.
for _, v := range reapCheckerPods {
if v.Labels["kuberhealthy-check-name"] == checkName {
- if time.Now().Sub(getPodCompletedTime(v)) > minCheckPodAge {
+ podTerminated, podTerminatedTime := getPodCompletedTime(v)
+ if podTerminated {
+ if time.Now().Sub(podTerminatedTime) > minCheckPodAge {
allCheckPods = append(allCheckPods, v)
}
}
}
+ }
return allCheckPods
}
@@ -336,17 +341,20 @@ func khJobDelete(client *khjobcrd.KHJobV1Client) error {
return nil
}
-// getPodCompletedTime returns the container with the latest finished time.
-func getPodCompletedTime(pod v1.Pod) time.Time {
+// getPodCompletedTime returns a boolean to ensure container terminated state exists and returns containers' latest finished time
+func getPodCompletedTime(pod v1.Pod) (bool, time.Time) {
var podCompletedTime time.Time
for _, cs := range pod.Status.ContainerStatuses {
-
+ if cs.State.Terminated != nil {
finishedTime := cs.State.Terminated.FinishedAt
if finishedTime.After(podCompletedTime) {
podCompletedTime = finishedTime.Time
}
+ } else {
+ return false, podCompletedTime
+ }
}
- return podCompletedTime
+ return true, podCompletedTime
}
