diff --git a/src/output_raytune.ipynb b/src/output_raytune.ipynb index 18293c4..7230e7e 100644 --- a/src/output_raytune.ipynb +++ b/src/output_raytune.ipynb @@ -594,9 +594,9 @@ "

Tune Status

\n", " \n", "\n", - "\n", - "\n", - "\n", + "\n", + "\n", + "\n", "\n", "
Current time:2026-01-18 17:37:46
Running for: 00:12:03.55
Memory: 16.5/119.7 GiB
Current time:2026-01-18 17:38:46
Running for: 00:13:03.82
Memory: 14.3/119.7 GiB
\n", " \n", @@ -610,12 +610,14 @@ "

Messages

\n", " \n", " \n", - " Number of errored trials: 1
\n", + " Number of errored trials: 3
\n", "\n", "\n", "\n", "\n", "\n", + "\n", + "\n", "\n", "
Trial name # failureserror file
trainable_paddle_ocr_36ae4d11 1/tmp/ray/session_2026-01-18_17-25-40_347373_1281294/artifacts/2026-01-18_17-25-43/trainable_paddle_ocr_2026-01-18_17-25-43/driver_artifacts/trainable_paddle_ocr_36ae4d11_1_text_det_box_thresh=0.5847,text_det_thresh=0.2571,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_17-25-43/error.txt
trainable_paddle_ocr_2312d29c 1/tmp/ray/session_2026-01-18_17-25-40_347373_1281294/artifacts/2026-01-18_17-25-43/trainable_paddle_ocr_2026-01-18_17-25-43/driver_artifacts/trainable_paddle_ocr_2312d29c_2_text_det_box_thresh=0.0312,text_det_thresh=0.0223,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_17-25-44/error.txt
trainable_paddle_ocr_5b7b8e02 1/tmp/ray/session_2026-01-18_17-25-40_347373_1281294/artifacts/2026-01-18_17-25-43/trainable_paddle_ocr_2026-01-18_17-25-43/driver_artifacts/trainable_paddle_ocr_5b7b8e02_3_text_det_box_thresh=0.5954,text_det_thresh=0.0707,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_17-31-48/error.txt
\n", "\n", @@ -650,9 +652,11 @@ "classify use_doc_unwarping \n", "\n", "\n", - "trainable_paddle_ocr_2312d29cRUNNING 192.168.65.140:1282844 0.0311783 0.022272400.141805False True False \n", - "trainable_paddle_ocr_5b7b8e02RUNNING 192.168.65.140:1285648 0.595412 0.070652200.132174True FalseTrue \n", + "trainable_paddle_ocr_b3243c8aRUNNING 192.168.65.140:1288101 0.360789 0.499551 00.115115False True False \n", + "trainable_paddle_ocr_7a4a43b0PENDING 0.0727848 0.237729 00.33623 True FalseTrue \n", "trainable_paddle_ocr_36ae4d11ERROR 192.168.65.140:1282742 0.58473 0.257102 00.634955False True False \n", + "trainable_paddle_ocr_2312d29cERROR 192.168.65.140:1282844 0.0311783 0.022272400.141805False True False \n", + "trainable_paddle_ocr_5b7b8e02ERROR 192.168.65.140:1285648 0.595412 0.070652200.132174True FalseTrue \n", "\n", "\n", " \n", @@ -770,6 +774,88 @@ "text": [ "\u001b[36m(trainable_paddle_ocr pid=1285648)\u001b[0m [2026-01-18 17:32:19,397 E 1285648 1285683] core_worker_process.cc:842: Failed to establish connection to the metrics exporter agent. Metrics will not be exported. Exporter agent status: RpcError: Running out of retries to initialize the metrics agent. rpc_code: 14\u001b[32m [repeated 20x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 17:38:37,341\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_2312d29c\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1282844, ip=192.168.65.140, actor_id=845cd8594f8ace3d960b90e501000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1281294/4208751894.py\", line 31, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.report` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 17:38:46,519\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_5b7b8e02\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1285648, ip=192.168.65.140, actor_id=b8478e34aea747352febbe0801000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1281294/4208751894.py\", line 31, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.report` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] } ], "source": [