diff --git "a/wandb/run-20240917_140929-fx32bo52/logs/debug-internal.log" "b/wandb/run-20240917_140929-fx32bo52/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240917_140929-fx32bo52/logs/debug-internal.log" @@ -0,0 +1,2556 @@ +2024-09-17 14:09:29,487 INFO StreamThr :1271339 [internal.py:wandb_internal():85] W&B internal server running at pid: 1271339, started at: 2024-09-17 14:09:29.483532 +2024-09-17 14:09:29,491 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status +2024-09-17 14:09:29,505 INFO WriterThread:1271339 [datastore.py:open_for_write():87] open: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/run-fx32bo52.wandb +2024-09-17 14:09:29,514 DEBUG SenderThread:1271339 [sender.py:send():391] send: header +2024-09-17 14:09:29,516 DEBUG SenderThread:1271339 [sender.py:send():391] send: run +2024-09-17 14:09:29,908 INFO SenderThread:1271339 [dir_watcher.py:__init__():211] watching files in: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files +2024-09-17 14:09:29,908 INFO SenderThread:1271339 [sender.py:_start_run_threads():1200] run started: fx32bo52 with start time 1726578569.485463 +2024-09-17 14:09:29,914 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: run_start +2024-09-17 14:09:30,018 DEBUG HandlerThread:1271339 [system_info.py:__init__():26] System info init +2024-09-17 14:09:30,018 DEBUG HandlerThread:1271339 [system_info.py:__init__():41] System info init done +2024-09-17 14:09:30,018 INFO HandlerThread:1271339 [system_monitor.py:start():194] Starting system monitor +2024-09-17 14:09:30,019 INFO SystemMonitor:1271339 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-17 14:09:30,020 INFO HandlerThread:1271339 [system_monitor.py:probe():214] Collecting system info +2024-09-17 14:09:30,021 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started cpu monitoring +2024-09-17 14:09:30,023 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started disk monitoring +2024-09-17 14:09:30,027 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started gpu monitoring +2024-09-17 14:09:30,028 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started memory monitoring +2024-09-17 14:09:30,030 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started network monitoring +2024-09-17 14:09:30,062 DEBUG HandlerThread:1271339 [system_info.py:probe():152] Probing system +2024-09-17 14:09:30,066 DEBUG HandlerThread:1271339 [gitlib.py:_init_repo():56] git repository is invalid +2024-09-17 14:09:30,066 DEBUG HandlerThread:1271339 [system_info.py:probe():200] Probing system done +2024-09-17 14:09:30,066 DEBUG HandlerThread:1271339 [system_monitor.py:probe():223] {'os': 'Linux-4.18.0-513.24.1.el8_9.x86_64-x86_64-with-glibc2.28', 'python': '3.11.4', 'heartbeatAt': '2024-09-17T13:09:30.062514', 'startedAt': '2024-09-17T13:09:29.455485', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '', 'codePathLocal': None, 'host': 'codon-gpu-014.ebi.ac.uk', 'username': 'amrufai', 'executable': '/nfs/production/literature/amina-mardiyyah/envs/llm-prompt/bin/python', 'cpu_count': 48, 'cpu_count_logical': 48, 'cpu_freq': {'current': 2758.5387083333335, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 809.858, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 47.760292053222656, 'used': 15.848091125488281}}, 'gpu': 'NVIDIA A100 80GB PCIe', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100 80GB PCIe', 'memory_total': 85899345920}], 'memory': {'total': 502.83758544921875}} +2024-09-17 14:09:30,066 INFO HandlerThread:1271339 [system_monitor.py:probe():224] Finished collecting system info +2024-09-17 14:09:30,066 INFO HandlerThread:1271339 [system_monitor.py:probe():227] Publishing system info +2024-09-17 14:09:30,070 INFO HandlerThread:1271339 [system_monitor.py:probe():229] Finished publishing system info +2024-09-17 14:09:30,114 DEBUG SenderThread:1271339 [sender.py:send():391] send: files +2024-09-17 14:09:30,115 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-metadata.json with policy now +2024-09-17 14:09:30,801 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: python_packages +2024-09-17 14:09:30,802 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: python_packages +2024-09-17 14:09:30,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:09:30,808 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:09:30,808 DEBUG SenderThread:1271339 [sender.py:send():391] send: telemetry +2024-09-17 14:09:30,808 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:09:30,822 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: pause +2024-09-17 14:09:30,822 INFO HandlerThread:1271339 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-17 14:09:30,822 INFO HandlerThread:1271339 [system_monitor.py:finish():203] Stopping system monitor +2024-09-17 14:09:30,823 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-17 14:09:30,823 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-17 14:09:30,823 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-17 14:09:30,825 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined cpu monitor +2024-09-17 14:09:30,826 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined disk monitor +2024-09-17 14:09:30,885 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined gpu monitor +2024-09-17 14:09:30,885 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined memory monitor +2024-09-17 14:09:30,886 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined network monitor +2024-09-17 14:09:30,912 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_created():271] file/dir created: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-metadata.json +2024-09-17 14:09:30,913 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_created():271] file/dir created: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/requirements.txt +2024-09-17 14:09:31,019 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:09:32,629 INFO wandb-upload_0:1271339 [upload_job.py:push():130] Uploaded file /tmp/tmposow_s3iwandb/pphmy8dt-wandb-metadata.json +2024-09-17 14:09:35,020 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:09:40,804 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:09:40,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:09:45,802 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:09:45,803 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:09:45,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:09:50,804 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:09:51,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:09:57,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:00,802 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:10:00,803 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:10:00,843 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:10:02,965 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:03,945 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/config.yaml +2024-09-17 14:10:08,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:10,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:10:14,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:15,802 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:10:15,803 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:10:19,990 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:20,804 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:10:25,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:30,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:10:30,803 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:10:30,843 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:10:31,027 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:36,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:40,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:10:41,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:45,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:10:45,804 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:10:47,000 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:50,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:10:52,808 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:10:58,807 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:00,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:11:00,804 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:11:00,844 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:11:03,964 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:09,807 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:10,807 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:11:14,807 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:15,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:11:15,804 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:11:19,937 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:20,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:11:21,295 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: resume +2024-09-17 14:11:21,295 INFO HandlerThread:1271339 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-17 14:11:21,295 INFO HandlerThread:1271339 [system_monitor.py:start():194] Starting system monitor +2024-09-17 14:11:21,296 INFO SystemMonitor:1271339 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-17 14:11:21,297 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started cpu monitoring +2024-09-17 14:11:21,299 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started disk monitoring +2024-09-17 14:11:21,302 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started gpu monitoring +2024-09-17 14:11:21,305 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started memory monitoring +2024-09-17 14:11:21,308 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started network monitoring +2024-09-17 14:11:21,354 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: pause +2024-09-17 14:11:21,354 INFO HandlerThread:1271339 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-17 14:11:21,354 INFO HandlerThread:1271339 [system_monitor.py:finish():203] Stopping system monitor +2024-09-17 14:11:21,354 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-17 14:11:21,354 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-17 14:11:21,354 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-17 14:11:21,358 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined cpu monitor +2024-09-17 14:11:21,359 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined disk monitor +2024-09-17 14:11:21,418 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined gpu monitor +2024-09-17 14:11:21,419 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined memory monitor +2024-09-17 14:11:21,419 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined network monitor +2024-09-17 14:11:21,420 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:11:23,084 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: resume +2024-09-17 14:11:23,084 INFO HandlerThread:1271339 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-17 14:11:23,085 INFO HandlerThread:1271339 [system_monitor.py:start():194] Starting system monitor +2024-09-17 14:11:23,085 INFO SystemMonitor:1271339 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-17 14:11:23,087 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started cpu monitoring +2024-09-17 14:11:23,089 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started disk monitoring +2024-09-17 14:11:23,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: pause +2024-09-17 14:11:23,091 INFO HandlerThread:1271339 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-17 14:11:23,091 INFO HandlerThread:1271339 [system_monitor.py:finish():203] Stopping system monitor +2024-09-17 14:11:23,091 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started gpu monitoring +2024-09-17 14:11:23,092 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-17 14:11:23,092 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-17 14:11:23,092 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-17 14:11:23,095 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined cpu monitor +2024-09-17 14:11:23,097 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined disk monitor +2024-09-17 14:11:23,152 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined gpu monitor +2024-09-17 14:11:23,154 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:11:25,155 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:28,595 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: resume +2024-09-17 14:11:28,596 INFO HandlerThread:1271339 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-17 14:11:28,596 INFO HandlerThread:1271339 [system_monitor.py:start():194] Starting system monitor +2024-09-17 14:11:28,596 INFO SystemMonitor:1271339 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-17 14:11:28,598 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started cpu monitoring +2024-09-17 14:11:28,601 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started disk monitoring +2024-09-17 14:11:28,602 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started gpu monitoring +2024-09-17 14:11:28,604 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started memory monitoring +2024-09-17 14:11:28,605 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started network monitoring +2024-09-17 14:11:29,030 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_created():271] file/dir created: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 14:11:30,646 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:31,036 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 14:11:35,647 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:35,932 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:11:35,933 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:11:35,933 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:11:36,277 DEBUG SenderThread:1271339 [sender.py:send():391] send: config +2024-09-17 14:11:36,278 DEBUG SenderThread:1271339 [sender.py:send():391] send: telemetry +2024-09-17 14:11:36,278 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:11:36,279 DEBUG SenderThread:1271339 [sender.py:send():391] send: telemetry +2024-09-17 14:11:36,279 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:11:36,279 WARNING SenderThread:1271339 [sender.py:send_metric():1417] Seen metric with glob (shouldn't happen) +2024-09-17 14:11:36,279 DEBUG SenderThread:1271339 [sender.py:send():391] send: telemetry +2024-09-17 14:11:36,279 DEBUG SenderThread:1271339 [sender.py:send():391] send: telemetry +2024-09-17 14:11:36,279 DEBUG SenderThread:1271339 [sender.py:send():391] send: config +2024-09-17 14:11:39,045 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 14:11:40,937 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:42,048 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/config.yaml +2024-09-17 14:11:45,801 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:11:46,927 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:47,753 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:11:47,754 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:11:52,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:11:55,802 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:11:58,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:02,753 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:12:02,754 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:12:04,910 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:05,802 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:12:09,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:15,802 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:12:15,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:17,753 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:12:17,754 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:12:20,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:25,802 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:12:26,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:28,606 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-17 14:12:28,609 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:12:31,929 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:32,753 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:12:32,754 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:12:35,802 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:12:37,929 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:42,929 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:45,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:12:47,753 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:12:47,754 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:12:47,962 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:53,932 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:12:55,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:12:58,611 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:12:59,613 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:02,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:13:02,754 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:13:04,905 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:05,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:13:09,931 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:14,932 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:15,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:13:17,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:13:17,755 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:13:19,985 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:25,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:13:25,931 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:28,613 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:13:30,931 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:32,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:13:32,755 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:13:35,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:13:35,932 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:41,932 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:45,803 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:13:46,932 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:47,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:13:47,755 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:13:51,932 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:55,804 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:13:57,933 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:13:58,616 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:14:02,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:14:02,755 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:14:03,889 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:05,804 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:14:08,933 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:13,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:15,804 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:14:17,755 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:14:17,756 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:14:19,913 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:24,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:25,804 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:14:28,618 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:14:30,620 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:32,766 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:14:32,767 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:14:35,804 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:14:35,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:41,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:45,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:14:46,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:47,755 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:14:47,756 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:14:51,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:55,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:14:56,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:14:58,620 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:15:02,755 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:15:02,756 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:15:03,030 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:05,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:15:08,935 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:14,935 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:15,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:15:17,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:15:17,756 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:15:19,953 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:25,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:15:25,935 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:28,622 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:15:30,940 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:32,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:15:32,757 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:15:35,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:15:36,940 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:41,941 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:45,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:15:47,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:15:47,757 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:15:47,985 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:53,941 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:15:55,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:15:58,625 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:15:59,626 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:02,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:16:02,757 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:16:04,962 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:05,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:16:10,946 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:15,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:16:16,946 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:17,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:16:17,757 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:16:21,946 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:25,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:16:27,946 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:28,627 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:16:32,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:16:32,757 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:16:32,986 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:35,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:16:38,947 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:44,947 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:45,807 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:16:47,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:16:47,757 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:16:49,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:55,807 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:16:55,947 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:16:58,633 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:17:00,948 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:02,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:17:02,758 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:17:05,807 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:17:06,331 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:12,331 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:15,807 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:17:17,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:17:17,758 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:17:17,981 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:23,331 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:25,808 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:17:28,632 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:17:28,633 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:32,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:17:32,758 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:17:33,924 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:35,808 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:17:39,334 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:45,335 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:45,808 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:17:47,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:17:47,758 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:17:50,906 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:55,813 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:17:56,335 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:17:58,637 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:18:01,335 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:02,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:18:02,758 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:18:05,812 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:18:07,335 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:13,335 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:15,812 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:18:17,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:18:17,759 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:18:18,991 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:24,336 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:25,812 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:18:28,636 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:18:29,638 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:32,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:18:32,759 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:18:34,911 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:35,812 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:18:40,337 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:44,029 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:18:44,033 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:18:44,033 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:18:44,033 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:18:44,034 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:18:44,034 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:18:44,034 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:18:44,040 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:18:44,533 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_created():271] file/dir created: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:18:45,812 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:18:46,041 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:47,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:18:47,758 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:18:51,341 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:51,540 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/config.yaml +2024-09-17 14:18:55,812 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:18:57,337 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:18:58,638 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:19:02,337 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:02,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:19:02,759 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:19:05,812 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:19:07,337 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:12,338 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:15,813 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:19:17,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:19:17,759 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:19:17,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:23,341 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:25,813 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:19:28,640 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:19:28,645 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:32,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:19:32,759 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:19:33,942 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:35,813 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:19:39,339 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:44,339 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:45,813 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:19:47,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:19:47,759 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:19:49,933 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:55,340 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:19:55,843 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:19:58,643 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:19:58,746 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:19:58,751 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:19:58,752 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:19:58,752 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:19:58,752 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:19:58,752 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:19:58,752 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:19:58,753 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:19:58,753 DEBUG SenderThread:1271339 [sender.py:send():391] send: metric +2024-09-17 14:19:58,753 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:19:58,753 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:19:58,756 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:19:59,603 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:20:00,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:01,708 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 14:20:05,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:07,495 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:20:07,495 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:20:07,497 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:20:11,633 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:17,491 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:19,353 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:20:22,492 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:20:22,492 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:20:22,685 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:28,351 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:28,645 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:20:29,131 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/config.yaml +2024-09-17 14:20:29,353 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:20:34,342 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:37,492 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:20:37,493 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:20:39,354 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:20:40,342 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:45,342 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:49,354 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:20:51,343 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:52,492 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:20:52,493 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:20:57,343 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:20:58,648 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:20:59,354 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:21:02,343 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:07,344 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:07,492 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:21:07,493 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:21:09,354 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:21:12,344 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:18,344 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:19,358 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:21:22,492 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:21:22,493 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:21:23,708 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:28,650 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:21:29,344 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:29,358 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:21:34,344 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:37,492 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:21:37,493 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:21:39,344 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:39,360 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:21:45,344 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:49,360 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:21:50,345 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:52,493 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:21:52,494 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:21:56,345 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:21:58,652 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:21:59,360 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:22:01,345 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:06,345 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:07,493 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:22:07,494 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:22:09,364 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:22:12,345 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:17,346 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:19,364 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:22:22,493 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:22:22,494 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:22:22,938 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:28,346 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:28,654 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:22:29,365 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:22:33,347 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:37,494 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:22:37,495 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:22:38,643 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:39,365 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:22:44,346 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:49,365 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:22:50,346 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:52,493 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:22:52,494 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:22:55,347 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:22:58,657 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:22:59,365 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:23:00,347 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:05,347 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:07,494 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:23:07,495 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:23:09,365 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:23:11,347 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:17,348 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:19,365 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:23:22,494 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:23:22,495 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:23:22,667 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:28,348 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:28,659 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:23:29,366 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:23:33,349 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:37,494 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:23:37,495 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:23:38,669 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:39,366 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:23:44,348 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:49,348 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:49,366 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:23:52,496 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:23:52,497 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:23:54,348 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:58,665 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:23:59,349 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:23:59,366 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:24:05,349 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:07,496 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:24:07,497 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:24:09,366 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:24:10,349 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:15,349 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:19,367 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:24:20,350 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:22,496 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:24:22,497 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:24:26,350 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:28,664 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:24:29,367 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:24:31,350 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:36,353 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:37,504 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:24:37,505 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:24:39,367 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:24:42,354 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:47,354 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:49,367 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:24:52,354 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:52,498 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:24:52,499 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:24:57,354 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:24:58,669 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:24:59,367 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:25:02,355 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:07,498 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:25:07,499 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:25:07,708 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:09,368 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:25:13,355 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:18,355 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:19,368 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:25:22,498 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:25:22,499 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:25:23,648 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:28,668 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:25:28,673 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:29,368 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:25:34,355 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:37,498 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:25:37,499 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:25:39,368 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:25:40,355 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:45,356 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:49,373 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:25:51,356 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:52,499 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:25:52,500 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:25:56,356 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:25:58,670 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:25:59,368 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:26:01,357 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:07,357 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:07,499 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:26:07,500 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:26:09,368 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:26:12,357 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:17,357 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:19,368 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:26:22,499 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:26:22,500 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:26:22,698 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:28,358 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:28,673 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:26:29,369 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:26:33,358 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:37,500 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:26:37,500 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:26:38,710 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:39,369 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:26:44,358 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:49,369 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:26:50,358 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:52,500 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:26:52,500 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:26:55,358 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:26:58,675 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:26:59,369 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:27:00,359 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:06,359 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:07,500 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:27:07,501 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:27:09,369 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:27:11,359 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:16,359 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:19,369 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:27:21,359 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:22,500 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:27:22,501 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:27:27,360 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:27,539 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:27:27,541 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:27:27,542 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:27:27,545 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:27:27,617 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:27:28,677 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:27:29,370 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:27:32,360 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:37,500 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:27:37,501 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:27:38,337 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:39,371 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:27:43,359 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:48,362 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:49,370 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:27:52,500 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:27:52,501 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:27:53,716 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:58,679 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:27:59,360 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:27:59,371 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:28:04,360 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:07,500 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:28:07,501 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:28:09,372 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:28:10,360 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:15,361 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:19,371 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:28:21,361 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:22,501 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:28:22,501 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:28:26,361 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:28,682 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:28:29,371 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:28:32,361 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:37,361 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:37,501 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:28:37,501 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:28:39,384 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:28:42,166 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:28:42,168 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:28:42,169 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:28:42,172 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:28:42,694 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:28:43,173 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:43,696 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 14:28:48,174 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:50,444 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:28:52,899 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:28:52,900 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:28:54,103 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:28:58,684 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:28:59,686 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:01,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:29:04,772 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:07,899 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:29:07,900 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:29:10,040 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:11,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:29:15,764 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:20,764 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:21,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:29:22,900 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:29:22,901 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:29:26,094 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:28,686 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:29:31,689 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:31,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:29:36,764 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:37,900 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:29:37,901 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:29:41,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:29:41,764 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:46,765 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:51,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:29:52,765 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:52,900 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:29:52,901 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:29:57,765 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:29:58,689 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:30:01,755 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:30:03,765 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:07,900 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:30:07,901 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:30:09,097 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:11,755 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:30:14,769 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:20,769 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:21,755 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:30:22,900 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:30:22,901 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:30:26,111 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:28,691 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:30:31,693 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:31,755 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:30:36,771 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:37,900 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:30:37,901 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:30:41,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:30:42,771 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:47,772 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:51,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:30:52,772 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:30:52,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:30:52,901 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:30:58,693 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:30:58,698 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:01,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:31:03,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:07,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:31:07,902 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:31:09,070 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:11,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:31:14,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:19,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:21,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:31:22,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:31:22,902 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:31:25,117 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:28,697 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:31:30,699 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:31,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:31:35,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:37,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:31:37,902 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:31:41,126 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:41,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:31:46,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:51,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:31:52,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:52,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:31:52,902 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:31:57,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:31:58,698 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:32:01,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:32:02,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:07,780 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:07,902 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:32:07,903 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:32:11,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:32:13,778 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:18,779 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:21,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:32:22,902 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:32:22,902 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:32:24,080 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:28,700 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:32:29,702 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:31,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:32:34,782 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:37,902 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:32:37,903 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:32:40,082 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:41,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:32:45,783 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:51,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:32:51,782 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:52,902 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:32:52,903 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:32:56,782 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:32:58,702 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:33:01,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:33:01,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:07,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:07,902 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:33:07,903 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:33:11,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:33:12,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:17,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:21,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:33:22,902 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:33:22,903 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:33:23,132 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:28,705 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:33:28,709 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:31,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:33:33,785 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:37,902 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:33:37,903 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:33:39,095 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:41,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:33:44,785 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:49,786 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:51,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:33:52,903 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:33:52,904 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:33:55,076 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:33:58,707 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:34:00,709 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:01,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:34:05,788 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:07,903 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:34:07,904 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:34:11,102 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:11,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:34:16,788 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:21,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:34:22,788 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:22,903 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:34:22,904 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:34:27,788 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:28,709 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:34:31,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:34:33,788 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:37,903 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:34:37,904 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:34:39,060 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:41,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:34:44,789 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:49,789 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:51,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:34:52,903 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:34:52,904 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:34:55,128 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:34:58,712 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:35:00,714 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:01,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:35:05,790 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:07,904 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:35:07,905 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:35:11,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:11,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:35:16,790 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:21,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:35:22,790 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:22,904 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:35:22,905 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:35:27,790 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:28,714 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:35:31,761 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:35:32,790 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:37,793 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:37,904 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:35:37,905 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:35:41,761 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:35:43,794 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:48,793 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:51,761 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:35:52,904 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:35:52,905 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:35:54,039 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:35:58,717 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:35:59,718 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:01,766 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:36:04,794 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:07,904 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:36:07,905 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:36:09,693 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:36:09,695 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:36:09,696 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:36:09,702 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:36:10,674 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:36:10,703 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:11,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:36:15,795 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:21,766 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:36:21,794 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:22,904 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:36:22,905 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:36:26,794 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:28,718 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:36:31,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:36:32,794 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:37,795 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:37,904 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:36:37,905 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:36:41,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:36:42,795 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:47,795 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:51,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:36:52,905 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:36:52,906 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:36:53,102 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:36:58,720 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:36:58,724 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:01,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:37:03,796 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:07,905 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:37:07,906 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:37:09,316 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:11,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:37:14,796 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:19,796 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:21,794 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:37:22,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:37:22,922 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:37:24,845 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:24,846 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:37:24,848 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:37:24,849 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:37:24,852 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:37:25,753 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:37:27,873 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 14:37:28,723 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:37:30,724 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:33,602 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:37:36,605 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:41,606 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:44,419 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:37:44,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:37:44,432 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:37:46,671 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:52,412 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:54,419 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:37:57,412 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:37:58,725 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:37:59,427 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:37:59,428 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:38:02,644 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:04,419 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:38:08,412 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:13,413 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:14,420 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:38:14,427 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:38:14,428 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:38:18,593 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:24,413 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:24,420 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:38:28,731 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:38:29,413 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:29,427 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:38:29,428 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:38:34,413 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:34,420 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:38:40,413 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:44,420 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:38:44,427 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:38:44,428 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:38:45,617 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:51,414 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:54,421 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:38:57,414 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:38:58,730 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:38:59,428 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:38:59,428 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:39:02,577 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:04,421 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:39:08,415 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:13,415 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:14,421 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:39:14,428 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:39:14,429 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:39:18,631 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:24,417 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:24,422 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:39:28,735 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:39:29,417 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:29,428 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:39:29,429 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:39:34,417 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:34,422 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:39:39,418 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:44,418 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:44,422 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:39:44,428 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:39:44,429 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:39:49,418 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:54,419 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:54,422 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:39:58,734 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:39:59,418 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:39:59,428 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:39:59,429 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:40:04,419 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:04,423 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:40:09,419 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:14,419 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:14,423 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:40:14,428 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:40:14,429 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:40:19,420 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:24,421 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:24,424 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:40:28,737 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:40:29,429 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:40:29,429 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:40:29,601 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:34,424 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:40:35,420 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:40,423 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:44,425 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:40:44,429 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:40:44,430 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:40:45,577 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:51,424 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:54,424 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:40:57,424 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:40:58,739 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:40:59,429 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:40:59,430 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:41:02,559 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:04,425 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:41:08,425 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:14,425 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:14,425 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:41:14,429 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:41:14,430 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:41:19,425 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:24,425 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:24,426 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:41:28,741 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:41:29,429 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:41:29,430 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:41:29,648 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:34,426 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:41:35,425 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:41,426 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:44,426 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:41:44,430 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:41:44,430 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:41:46,638 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:52,426 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:54,427 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:41:57,427 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:41:58,743 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:41:59,430 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:41:59,431 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:42:02,634 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:04,426 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:42:08,428 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:14,428 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:14,429 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:42:14,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:42:14,431 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:42:20,427 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:24,426 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:42:25,429 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:28,746 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:42:29,430 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:42:29,431 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:42:30,703 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:34,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:42:36,433 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:42,429 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:44,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:42:44,432 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:42:44,472 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:42:47,589 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:53,432 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:54,430 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:42:58,748 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:42:58,753 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:42:59,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:42:59,432 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:43:04,430 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:04,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:43:09,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:14,430 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:43:14,432 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:43:14,433 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:43:14,575 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:20,435 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:24,430 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:43:26,435 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:28,751 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:43:29,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:43:29,432 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:43:31,573 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:34,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:43:37,435 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:42,436 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:44,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:43:44,432 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:43:44,433 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:43:47,589 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:53,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:54,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:43:58,753 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:43:58,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:43:59,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:43:59,432 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:44:04,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:44:04,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:09,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:14,432 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:44:14,433 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:44:14,472 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:44:14,573 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:20,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:24,443 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:44:25,446 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:28,755 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:44:29,432 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:44:29,433 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:44:30,559 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:34,439 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:44:36,443 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:42,444 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:44,432 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:44:44,433 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:44:44,473 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:44:47,571 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:53,444 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:54,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:44:55,544 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:44:55,547 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:44:55,547 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:44:55,551 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:44:55,782 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:44:58,553 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:44:58,757 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:44:59,432 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:44:59,433 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:45:03,569 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:04,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:45:09,444 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:14,432 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:45:14,433 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:45:14,473 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:45:14,651 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:20,444 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:24,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:45:26,444 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:28,759 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:45:29,432 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:45:29,433 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:45:31,649 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:34,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:45:37,445 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:42,445 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:44,433 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:45:44,433 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:45:44,473 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:45:47,576 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:53,446 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:54,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:45:58,763 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:45:58,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:45:59,433 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:45:59,434 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:46:04,440 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:46:04,445 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:09,446 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:10,492 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:46:10,495 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:46:10,495 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:46:10,499 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:46:10,858 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:46:11,859 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 14:46:14,500 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:19,501 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:19,504 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:46:19,505 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:46:19,507 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:46:24,716 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:28,764 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:46:29,766 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:30,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:46:34,504 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:46:34,505 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:46:35,703 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:40,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:46:40,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:45,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:49,504 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:46:49,505 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:46:50,965 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:46:50,982 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:56,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:46:58,767 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:47:00,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:47:01,980 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:04,504 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:47:04,505 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:47:07,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:10,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:47:13,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:18,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:19,504 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:47:19,505 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:47:20,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:47:24,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:28,769 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:47:30,771 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:30,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:47:34,505 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:47:34,506 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:47:35,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:40,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:47:40,980 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:45,982 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:49,505 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:47:49,506 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:47:50,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:47:50,982 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:55,982 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:47:58,771 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:48:00,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:48:00,982 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:04,505 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:48:04,506 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:48:05,982 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:10,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:48:11,982 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:16,983 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:19,505 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:48:19,506 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:48:20,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:48:22,983 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:28,773 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:48:28,778 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:30,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:48:33,983 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:34,505 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:48:34,506 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:48:38,983 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:40,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:48:44,984 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:49,505 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:48:49,506 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:48:50,666 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:50,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:48:55,984 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:48:58,776 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:49:00,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:49:01,984 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:04,506 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:49:04,506 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:49:06,984 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:10,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:49:12,985 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:17,985 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:19,506 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:49:19,507 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:49:20,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:49:23,985 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:28,778 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:49:29,780 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:30,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:49:34,506 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:49:34,507 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:49:35,665 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:40,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:49:40,986 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:45,986 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:49,506 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:49:49,507 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:49:50,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:49:50,986 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:55,986 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:49:58,780 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:50:00,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:50:00,986 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:04,506 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:50:04,507 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:50:05,987 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:10,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:50:11,986 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:16,987 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:19,506 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:50:19,507 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:50:20,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:50:22,987 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:27,987 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:28,782 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:50:30,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:50:33,988 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:34,507 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:50:34,508 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:50:38,988 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:40,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:50:43,988 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:49,507 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:50:49,508 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:50:49,710 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:50,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:50:54,988 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:50:58,785 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:51:00,787 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:00,976 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:51:04,507 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:51:04,508 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:51:05,988 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:10,976 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:51:10,988 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:15,989 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:19,507 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:51:19,508 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:51:20,976 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:51:21,989 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:26,989 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:28,787 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:51:30,976 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:51:32,989 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:34,507 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:51:34,508 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:51:37,990 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:40,977 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:51:42,997 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:48,996 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:49,507 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:51:49,508 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:51:50,977 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:51:54,997 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:51:58,789 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:52:00,791 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:00,977 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:52:04,508 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:52:04,509 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:52:05,996 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:10,977 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:52:10,997 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:15,997 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:19,508 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:52:19,509 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:52:20,977 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:52:20,997 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:25,997 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:28,792 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:52:30,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:52:31,997 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:34,508 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:52:34,509 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:52:36,998 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:40,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:52:42,998 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:47,998 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:49,508 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:52:49,509 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:52:50,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:52:53,999 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:52:58,794 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:52:59,796 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:00,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:53:04,508 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:53:04,509 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:53:05,727 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:10,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:53:10,998 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:15,999 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:19,509 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:53:19,510 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:53:20,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:53:21,999 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:27,005 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:28,798 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:53:30,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:53:33,004 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:34,509 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:53:34,510 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:53:38,004 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:40,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:53:43,008 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:44,019 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:53:44,021 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:53:44,022 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:53:44,025 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:53:44,806 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:53:49,006 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:49,509 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:53:49,510 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:53:50,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:53:55,006 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:53:58,798 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:54:00,800 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:00,982 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:54:04,509 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:54:04,510 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:54:06,006 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:10,980 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:54:11,006 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:16,007 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:19,509 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:54:19,510 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:54:20,980 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:54:22,007 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:28,007 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:28,800 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:54:30,980 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:54:33,007 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:34,510 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:54:34,511 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:54:38,008 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:40,980 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:54:43,009 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:49,008 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:49,509 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:54:49,510 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:54:50,980 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:54:55,009 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:54:58,511 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 14:54:58,514 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 14:54:58,514 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 14:54:58,517 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 14:54:58,803 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:54:58,882 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 14:54:59,886 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 14:55:00,805 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:05,806 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:07,247 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:55:07,249 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:55:07,249 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:55:11,376 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:16,377 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:18,768 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:55:21,770 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:22,248 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:55:22,248 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:55:27,755 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:28,768 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:55:28,805 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:55:32,808 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:37,247 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:55:37,248 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:55:38,468 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:38,768 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:55:43,756 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:48,757 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:48,768 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:55:52,248 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:55:52,249 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:55:53,758 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:55:58,776 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:55:58,808 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:55:58,812 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:04,761 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:07,248 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:56:07,249 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:56:08,770 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:56:10,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:16,759 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:18,771 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:56:22,248 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:56:22,249 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:56:22,418 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:27,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:28,771 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:56:28,810 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:56:32,812 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:37,248 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:56:37,249 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:56:38,395 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:38,771 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:56:43,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:48,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:48,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:56:52,248 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:56:52,249 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:56:53,761 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:56:58,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:56:58,812 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:56:58,817 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:04,761 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:07,248 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:57:07,249 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:57:08,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:57:10,761 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:16,762 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:18,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:57:21,762 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:22,249 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:57:22,250 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:57:27,762 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:28,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:57:28,815 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:57:32,817 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:37,249 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:57:37,250 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:57:38,375 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:38,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:57:43,763 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:48,764 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:48,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:57:52,253 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:57:52,254 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:57:53,765 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:57:58,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:57:58,817 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:57:58,821 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:04,765 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:07,253 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:58:07,254 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:58:08,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:58:09,766 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:15,766 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:18,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:58:20,766 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:22,254 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:58:22,254 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:58:26,766 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:28,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:58:28,819 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:58:31,821 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:37,254 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:58:37,255 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:58:37,385 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:38,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:58:42,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:47,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:48,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:58:52,261 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:58:52,262 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:58:53,508 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:58,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:58:58,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:58:58,821 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:59:03,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:07,257 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:59:07,258 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:59:08,768 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:08,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:59:14,768 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:18,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:59:19,768 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:22,257 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:59:22,258 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:59:25,768 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:28,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:59:28,824 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 14:59:30,826 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:36,769 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:37,257 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:59:37,258 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:59:38,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:59:42,769 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:47,770 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:48,776 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:59:52,257 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 14:59:52,258 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 14:59:53,481 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:58,769 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 14:59:58,777 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 14:59:58,826 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:00:03,769 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:07,258 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:00:07,259 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:00:08,770 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:08,779 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:00:13,770 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:18,770 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:18,779 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:00:22,258 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:00:22,259 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:00:24,770 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:28,779 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:00:28,832 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:00:29,834 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:35,771 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:37,258 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:00:37,259 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:00:38,779 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:00:41,771 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:46,771 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:48,779 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:00:52,262 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:00:52,263 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:00:52,464 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:57,772 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:00:58,780 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:00:58,831 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:01:02,833 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:07,260 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:01:07,261 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:01:08,491 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:08,780 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:01:13,773 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:18,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:18,782 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:01:22,261 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:01:22,261 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:01:24,774 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:28,783 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:01:28,836 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:01:29,838 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:35,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:37,261 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:01:37,261 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:01:38,783 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:01:41,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:46,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:48,783 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:01:51,775 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:52,261 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:01:52,262 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:01:57,776 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:01:58,783 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:01:58,835 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:02:02,837 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:07,261 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:02:07,262 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:02:08,472 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:08,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:02:13,776 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:18,776 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:18,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:02:22,261 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:02:22,262 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:02:24,776 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:28,254 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 15:02:28,256 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 15:02:28,256 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 15:02:28,261 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 15:02:28,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:02:28,800 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 15:02:28,837 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:02:29,839 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:35,776 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:37,261 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:02:37,262 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:02:38,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:02:40,776 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:45,777 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:48,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:02:51,777 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:52,261 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:02:52,262 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:02:57,777 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:02:58,784 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:02:58,839 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:03:02,841 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:07,262 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:03:07,262 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:03:08,549 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:08,785 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:03:13,778 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:18,778 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:18,785 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:03:22,262 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:03:22,262 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:03:24,778 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:28,785 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:03:28,842 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:03:29,843 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:35,779 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:37,262 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:03:37,263 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:03:38,786 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:03:40,779 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:43,047 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 15:03:43,050 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 15:03:43,050 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 15:03:43,053 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 15:03:43,869 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 15:03:43,870 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 15:03:45,779 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:50,780 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:51,879 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:03:52,548 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:03:52,549 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:03:56,691 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:03:58,844 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:04:01,846 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:03,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:04:06,961 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:07,555 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:04:07,555 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:04:12,954 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:13,959 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:04:18,954 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:22,549 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:04:22,550 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:04:23,955 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:23,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:04:28,847 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:04:29,955 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:33,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:04:35,955 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:37,550 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:04:37,551 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:04:41,955 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:43,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:04:47,955 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:52,549 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:04:52,550 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:04:53,713 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:04:53,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:04:58,849 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:04:58,854 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:03,955 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:03,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:05:07,550 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:05:07,550 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:05:08,955 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:13,956 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:13,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:05:18,957 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:22,550 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:05:22,550 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:05:23,957 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:23,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:05:28,852 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:05:28,957 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:33,958 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:33,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:05:37,550 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:05:37,551 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:05:39,958 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:43,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:05:44,958 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:49,958 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:52,550 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:05:52,551 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:05:53,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:05:54,959 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:05:58,854 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:06:00,959 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:03,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:06:06,959 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:07,550 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:06:07,551 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:06:11,959 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:13,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:06:16,959 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:22,550 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:06:22,551 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:06:22,911 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:23,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:06:27,960 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:28,856 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:06:32,960 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:33,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:06:37,551 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:06:37,551 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:06:38,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:43,960 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:43,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:06:48,960 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:52,551 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:06:52,552 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:06:53,961 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:06:53,972 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:06:58,858 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:06:58,961 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:03,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:07:04,961 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:07,551 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:07:07,552 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:07:09,962 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:13,972 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:07:14,962 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:19,962 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:22,551 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:07:22,552 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:07:23,972 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:07:25,962 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:28,861 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:07:30,963 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:33,972 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:07:35,963 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:37,552 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:07:37,553 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:07:41,963 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:43,972 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:07:46,963 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:52,552 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:07:52,553 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:07:52,767 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:53,972 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:07:57,964 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:07:58,865 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:08:02,964 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:03,982 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:08:07,552 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:08:07,553 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:08:08,783 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:13,964 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:13,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:08:18,965 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:22,552 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:08:22,553 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:08:23,964 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:23,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:08:28,865 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:08:28,965 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:33,965 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:33,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:08:37,554 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:08:37,555 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:08:38,965 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:43,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:43,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:08:48,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:52,555 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:08:52,555 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:08:53,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:08:53,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:08:58,867 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:08:59,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:03,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:09:04,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:07,555 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:09:07,555 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:09:09,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:13,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:09:14,966 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:19,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:22,555 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:09:22,556 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:09:23,976 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:09:24,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:28,870 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:09:30,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:33,981 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:09:35,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:37,555 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:09:37,556 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:09:40,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:43,976 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:09:46,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:52,555 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:09:52,556 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:09:52,841 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:53,977 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:09:57,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:09:58,872 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:10:02,969 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:03,977 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:10:07,555 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:10:07,556 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:10:08,862 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:13,968 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:13,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:10:18,969 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:22,556 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:10:22,557 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:10:23,969 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:23,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:10:28,874 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:10:29,969 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:33,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:10:34,970 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:37,556 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:10:37,557 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:10:39,971 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:43,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:10:44,978 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:50,972 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:52,560 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:10:52,560 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:10:53,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:10:55,972 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:10:58,877 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:11:00,972 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:03,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:11:06,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:07,558 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:11:07,559 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:11:11,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:13,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:11:14,553 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 15:11:14,555 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 15:11:14,556 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 15:11:14,559 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 15:11:14,941 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 15:11:17,560 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:22,558 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:11:22,559 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:11:22,707 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:23,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:11:27,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:28,879 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:11:33,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:33,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:11:37,558 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:11:37,559 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:11:38,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:43,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:43,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:11:49,973 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:52,559 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:11:52,559 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:11:53,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:11:54,976 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:11:58,881 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:12:00,974 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:03,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:12:05,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:07,559 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:12:07,559 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:12:10,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:13,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:12:16,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:21,975 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:22,559 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:12:22,559 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:12:23,979 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:12:26,976 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:28,883 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:12:29,524 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 15:12:29,527 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 15:12:29,527 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 15:12:29,530 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 15:12:30,014 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 15:12:32,139 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 15:12:32,531 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:37,532 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:38,236 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:12:38,237 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:12:38,238 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:12:43,407 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:48,408 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:49,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:12:53,235 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:12:53,236 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:12:54,172 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:58,886 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:12:59,173 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:12:59,178 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:13:05,173 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:08,235 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:13:08,236 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:13:09,178 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:13:10,177 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:16,175 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:19,178 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:13:21,175 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:23,236 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:13:23,236 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:13:27,175 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:28,888 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:13:29,179 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:13:32,175 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:38,176 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:38,236 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:13:38,237 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:13:39,179 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:13:43,176 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:49,175 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:49,179 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:13:53,236 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:13:53,237 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:13:54,176 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:58,890 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:13:59,176 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:13:59,180 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:14:04,176 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:08,236 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:14:08,237 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:14:09,176 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:09,180 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:14:14,177 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:19,180 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:14:20,176 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:23,236 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:14:23,237 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:14:25,177 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:28,893 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:14:29,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:14:30,177 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:35,177 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:38,237 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:14:38,238 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:14:39,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:14:41,177 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:46,178 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:49,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:14:51,178 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:53,237 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:14:53,238 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:14:56,178 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:14:58,898 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:14:59,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:15:01,178 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:06,178 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:08,237 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:15:08,238 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:15:09,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:15:12,179 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:18,179 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:19,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:15:23,179 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:23,237 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:15:23,238 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:15:28,897 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:15:28,899 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:29,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:15:34,179 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:38,237 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:15:38,238 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:15:39,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:15:40,179 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:45,180 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:49,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:15:51,180 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:53,237 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:15:53,238 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:15:56,180 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:15:58,902 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:15:59,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:16:02,180 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:07,181 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:08,238 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:16:08,239 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:16:09,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:16:13,181 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:18,181 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:19,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:16:23,238 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:16:23,239 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:16:23,408 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:28,902 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:16:28,906 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:29,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:16:34,181 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:38,238 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:16:38,239 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:16:39,181 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:39,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:16:44,182 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:49,182 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:49,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:16:53,243 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:16:53,243 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:16:54,182 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:58,904 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:16:59,183 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:16:59,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:17:04,183 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:08,241 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:17:08,242 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:17:09,183 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:09,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:17:14,183 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:19,184 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:19,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:17:23,241 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:17:23,242 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:17:25,184 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:28,906 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:17:29,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:17:30,184 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:36,184 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:38,241 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:17:38,242 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:17:39,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:17:41,184 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:46,185 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:49,190 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:17:51,185 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:53,242 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:17:53,243 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:17:56,186 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:17:58,909 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:17:59,190 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:18:01,186 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:07,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:08,245 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:18:08,246 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:18:09,190 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:18:13,186 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:18,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:19,190 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:18:23,245 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:18:23,245 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:18:23,383 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:28,911 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:18:28,915 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:29,191 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:18:34,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:38,245 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:18:38,246 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:18:39,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:39,190 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:18:44,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:49,190 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:18:50,187 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:53,245 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:18:53,246 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:18:56,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:18:58,913 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:18:59,191 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:19:01,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:06,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:08,245 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:19:08,246 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:19:09,195 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:19:12,188 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:17,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:19,195 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:19:22,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:23,246 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:19:23,246 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:19:28,189 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:28,915 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:19:29,195 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:19:33,192 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:38,246 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:19:38,246 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:19:38,428 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:39,195 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:19:44,190 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:49,190 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:49,195 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:19:53,246 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:19:53,247 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:19:55,190 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:19:58,918 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:19:59,196 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:20:00,191 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:00,392 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 15:20:00,393 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 15:20:00,394 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 15:20:00,397 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 15:20:01,171 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 15:20:06,191 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:08,246 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:20:08,247 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:20:09,196 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:20:11,191 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:16,191 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:19,196 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:20:22,191 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:23,246 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:20:23,247 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:20:27,191 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:28,920 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:20:29,196 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:20:32,191 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:37,194 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:38,246 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:20:38,247 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:20:39,197 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:20:43,192 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:48,192 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:49,197 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:20:53,192 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:53,246 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:20:53,247 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:20:58,193 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:20:58,922 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:20:59,197 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:21:03,193 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:08,246 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:21:08,247 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:21:08,481 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:09,197 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:21:14,193 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:15,287 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 15:21:15,290 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 15:21:15,290 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 15:21:15,294 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 15:21:16,245 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 15:21:16,246 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 15:21:19,296 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:24,297 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:24,574 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:21:24,576 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:21:24,576 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:21:28,925 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:21:29,926 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:34,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:35,913 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:21:39,572 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:21:39,573 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:21:40,754 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:45,900 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:45,905 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:21:50,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:54,572 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:21:54,573 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:21:55,906 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:21:56,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:21:58,927 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:22:01,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:05,906 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:22:07,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:09,572 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:22:09,573 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:22:13,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:15,906 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:22:18,901 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:23,902 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:24,572 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:22:24,573 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:22:25,907 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:22:28,931 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:22:28,935 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:34,902 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:35,911 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:22:39,572 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:22:39,573 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:22:40,772 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:45,906 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:45,911 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:22:50,906 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:54,573 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:22:54,574 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:22:55,907 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:22:55,912 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:22:58,932 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:23:00,911 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:05,911 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:05,912 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:23:09,573 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:23:09,574 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:23:10,912 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:15,912 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:15,914 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:23:20,913 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:24,573 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:23:24,574 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:23:25,918 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:25,924 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:23:28,934 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:23:31,918 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:35,919 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:23:36,919 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:39,573 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:23:39,574 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:23:42,919 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:45,919 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:23:47,919 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:52,919 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:54,573 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:23:54,574 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:23:55,919 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:23:57,919 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:23:58,936 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:24:02,927 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:05,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:24:08,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:09,573 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:24:09,574 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:24:14,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:15,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:24:19,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:24,574 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:24:24,574 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:24:25,776 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:25,922 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:24:28,938 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:24:30,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:35,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:35,923 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:24:39,574 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:24:39,575 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:24:41,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:45,922 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:24:47,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:52,922 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:54,574 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:24:54,575 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:24:55,922 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:24:58,922 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:24:58,941 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:25:04,922 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:05,922 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:25:09,574 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:25:09,575 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:25:10,751 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:15,922 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:15,924 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:25:20,923 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:24,575 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:25:24,576 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:25:25,924 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:25:25,926 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:28,943 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:25:31,924 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:35,925 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:25:36,924 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:39,574 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:25:39,575 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:25:42,924 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:45,926 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:25:47,926 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:53,925 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:54,575 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:25:54,575 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:25:55,925 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:25:58,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:25:58,945 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:26:04,925 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:05,924 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:26:09,575 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:26:09,576 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:26:10,743 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:15,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:15,929 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:26:21,926 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:24,575 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:26:24,576 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:26:25,924 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:26:26,927 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:28,948 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:26:32,926 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:35,927 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:26:37,927 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:39,575 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:26:39,576 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:26:43,927 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:45,925 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:26:48,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:54,575 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:26:54,576 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:26:54,765 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:26:55,925 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:26:58,950 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:26:59,952 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:05,926 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:27:05,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:09,575 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:27:09,576 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:27:11,928 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:15,926 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:27:16,929 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:22,929 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:24,579 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:27:24,580 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:27:25,935 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:27:27,937 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:28,952 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:27:33,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:35,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:27:38,937 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:39,580 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:27:39,580 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:27:44,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:45,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:27:49,937 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:54,579 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:27:54,580 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:27:55,769 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:27:55,934 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:27:58,955 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:28:00,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:05,938 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:05,940 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:28:09,580 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:28:09,581 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:28:11,938 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:15,935 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:28:16,938 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:22,938 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:24,580 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:28:24,581 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:28:25,935 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:28:27,939 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:28,957 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:28:33,939 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:35,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:28:38,940 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:39,583 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:28:39,584 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:28:43,940 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:45,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:28:47,667 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 15:28:47,669 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 15:28:47,670 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 15:28:47,673 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 15:28:48,219 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 15:28:49,674 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:54,581 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:28:54,582 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:28:54,760 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:28:55,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:28:58,959 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:28:59,960 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:05,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:29:05,941 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:09,581 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:29:09,582 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:29:10,941 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:15,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:29:16,941 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:21,942 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:24,581 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:29:24,582 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:29:25,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:29:27,942 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:28,961 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:29:32,942 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:35,936 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:29:38,942 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:39,582 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:29:39,582 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:29:43,942 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:45,937 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:29:49,943 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:54,585 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:29:54,585 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:29:55,798 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:29:55,937 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:29:58,965 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:30:00,967 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:02,423 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: partial_history +2024-09-17 15:30:02,426 DEBUG SenderThread:1271339 [sender.py:send():391] send: history +2024-09-17 15:30:02,426 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: summary_record +2024-09-17 15:30:02,429 INFO SenderThread:1271339 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-17 15:30:03,298 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/wandb-summary.json +2024-09-17 15:30:04,313 INFO Thread-12 :1271339 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised_no_date/wandb/run-20240917_140929-fx32bo52/files/output.log +2024-09-17 15:30:06,431 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:11,155 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:30:11,156 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:30:11,156 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:30:12,335 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:17,336 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:22,088 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:30:23,098 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:26,151 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:30:26,152 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:30:28,966 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:30:28,970 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:32,087 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:30:34,082 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:39,082 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:41,152 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:30:41,152 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:30:42,087 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:30:44,083 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:49,083 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:52,087 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:30:55,083 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:30:56,152 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:30:56,152 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:30:58,968 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:31:00,970 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:02,088 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:31:06,084 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:11,084 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:11,152 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:31:11,152 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:31:12,089 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:31:17,083 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:22,085 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:22,090 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:31:26,157 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:31:26,158 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:31:27,089 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:28,970 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:31:32,090 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:32,090 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:31:37,090 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:39,473 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: pause +2024-09-17 15:31:39,473 INFO HandlerThread:1271339 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-17 15:31:39,473 INFO HandlerThread:1271339 [system_monitor.py:finish():203] Stopping system monitor +2024-09-17 15:31:39,474 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-17 15:31:39,475 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-17 15:31:39,478 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined cpu monitor +2024-09-17 15:31:39,478 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined disk monitor +2024-09-17 15:31:39,515 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined gpu monitor +2024-09-17 15:31:39,515 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined memory monitor +2024-09-17 15:31:39,516 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined network monitor +2024-09-17 15:31:39,516 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:31:41,155 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:31:41,156 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:31:41,931 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: resume +2024-09-17 15:31:41,932 INFO HandlerThread:1271339 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-17 15:31:41,932 INFO HandlerThread:1271339 [system_monitor.py:start():194] Starting system monitor +2024-09-17 15:31:41,932 INFO SystemMonitor:1271339 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-17 15:31:41,934 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started cpu monitoring +2024-09-17 15:31:41,937 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started disk monitoring +2024-09-17 15:31:41,938 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started gpu monitoring +2024-09-17 15:31:41,939 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started memory monitoring +2024-09-17 15:31:41,942 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started network monitoring +2024-09-17 15:31:41,943 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: pause +2024-09-17 15:31:41,944 INFO HandlerThread:1271339 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-17 15:31:41,944 INFO HandlerThread:1271339 [system_monitor.py:finish():203] Stopping system monitor +2024-09-17 15:31:41,944 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined cpu monitor +2024-09-17 15:31:41,945 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-17 15:31:41,945 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-17 15:31:41,946 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-17 15:31:41,950 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined disk monitor +2024-09-17 15:31:42,008 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined gpu monitor +2024-09-17 15:31:42,008 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined memory monitor +2024-09-17 15:31:42,008 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined network monitor +2024-09-17 15:31:42,009 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:31:42,090 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:31:43,090 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:49,021 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: resume +2024-09-17 15:31:49,021 INFO HandlerThread:1271339 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-17 15:31:49,021 INFO HandlerThread:1271339 [system_monitor.py:start():194] Starting system monitor +2024-09-17 15:31:49,022 INFO SystemMonitor:1271339 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-17 15:31:49,023 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started cpu monitoring +2024-09-17 15:31:49,026 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started disk monitoring +2024-09-17 15:31:49,028 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started gpu monitoring +2024-09-17 15:31:49,029 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started memory monitoring +2024-09-17 15:31:49,031 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started network monitoring +2024-09-17 15:31:49,090 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:49,569 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: pause +2024-09-17 15:31:49,569 INFO HandlerThread:1271339 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-17 15:31:49,569 INFO HandlerThread:1271339 [system_monitor.py:finish():203] Stopping system monitor +2024-09-17 15:31:49,570 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined cpu monitor +2024-09-17 15:31:49,572 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined disk monitor +2024-09-17 15:31:49,572 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-17 15:31:49,572 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-17 15:31:49,572 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-17 15:31:49,634 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined gpu monitor +2024-09-17 15:31:49,635 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined memory monitor +2024-09-17 15:31:49,635 INFO HandlerThread:1271339 [interfaces.py:finish():200] Joined network monitor +2024-09-17 15:31:49,636 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:31:52,090 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:31:54,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:31:56,156 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:31:56,156 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:32:00,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:02,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:32:06,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:11,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:11,156 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:32:11,157 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:32:12,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:32:16,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:21,092 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:22,091 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:32:22,921 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: resume +2024-09-17 15:32:22,921 INFO HandlerThread:1271339 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-17 15:32:22,921 INFO HandlerThread:1271339 [system_monitor.py:start():194] Starting system monitor +2024-09-17 15:32:22,921 INFO SystemMonitor:1271339 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-17 15:32:22,923 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started cpu monitoring +2024-09-17 15:32:22,926 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started disk monitoring +2024-09-17 15:32:22,927 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started gpu monitoring +2024-09-17 15:32:22,928 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started memory monitoring +2024-09-17 15:32:22,929 INFO SystemMonitor:1271339 [interfaces.py:start():188] Started network monitoring +2024-09-17 15:32:26,092 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:31,093 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:32,651 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:32:32,656 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:32:32,657 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:32:36,804 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:42,650 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:32:42,654 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:47,656 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:32:47,657 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:32:47,878 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:52,651 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:32:53,654 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:32:58,655 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:02,650 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:33:02,656 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:33:02,657 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:33:03,840 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:09,655 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:12,651 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:33:14,656 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:17,657 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:33:17,657 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:33:19,867 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:22,651 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:33:22,930 DEBUG SystemMonitor:1271339 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-17 15:33:22,933 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats +2024-09-17 15:33:24,935 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:30,656 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:32,651 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:33:32,656 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:33:32,657 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:33:35,853 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:41,657 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:42,651 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:33:46,657 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:47,657 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: stop_status +2024-09-17 15:33:47,658 DEBUG SenderThread:1271339 [sender.py:send_request():418] send_request: stop_status +2024-09-17 15:33:51,851 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: status_report +2024-09-17 15:33:52,652 DEBUG HandlerThread:1271339 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-17 15:33:52,935 DEBUG SenderThread:1271339 [sender.py:send():391] send: stats