Api exception while sending a HTTP request to a miner
## Motivation
Following a recurring bug in the CI involving 'search_phrase_in_pod_log' automation function.
mining test would sometimes fail due to a 'kubernetes.client.rest.ApiException: (500)' exception.
this is caused because of an unhandled exception.
added a retry mechanism to handle those exceptions.
@@ -97,19 +97,31 @@ def check_for_restarted_pods(namespace, specific_deployment_name=''):
return restarted_pods
-def search_phrase_in_pod_log(pod_name, name_space, container_name, phrase, time_out=60, group=None):
-
- match = None
+def search_phrase_in_pod_log(pod_name, name_space, container_name, phrase, timeout=60, group=None, retries=3):
+ sleep_interval = 0.3
total_sleep = 0
+ retries_remaining = retries
while True:
- pod_logs = CoreV1ApiClient().read_namespaced_pod_log(name=pod_name,
- namespace=name_space,
- container=container_name)
+ try:
+ read_namespaced_pod_log = CoreV1ApiClient().read_namespaced_pod_log
+ pod_logs = read_namespaced_pod_log(name=pod_name, namespace=name_space, container=container_name)
+ except ApiException as e:
+ print(f"got an exception while reading pod log:\n{e}")
+ if retries_remaining <= 0:
+ raise ApiException(e)
+ retries_remaining -= 1
+ time.sleep(sleep_interval)
+ print(f"retrying, retries left:\n{retries_remaining}")
+ continue
+
match = re.search(phrase, pod_logs)
- if not match and total_sleep < time_out:
- time.sleep(1)
- total_sleep = total_sleep + 1
- else:
if match and group:
return match.group(group)
+ elif match:
return match
+ elif not match and total_sleep < timeout:
+ time.sleep(1)
+ total_sleep += 1
+ retries_remaining = retries
+ else:
+ return None
