diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..1c0b8af --- /dev/null +++ b/src/README.md @@ -0,0 +1,43 @@ +# Running Notebooks in Background + +## Option 1: Papermill (Recommended) + +Runs notebooks directly without conversion. + +```bash +pip install papermill +nohup papermill .ipynb output.ipynb > papermill.log 2>&1 & +``` + +Monitor: +```bash +tail -f papermill.log +``` + +## Option 2: Convert to Python Script + +```bash +jupyter nbconvert --to script .ipynb +nohup python .py > output.log 2>&1 & +``` + +**Note:** `%pip install` magic commands need manual removal before running as `.py` + +## Important Notes + +- Ray Tune notebooks require the OCR service running first (Docker) +- For Ray workers, imports must be inside trainable functions + +## Example: Ray Tune PaddleOCR + +```bash +# 1. Start OCR service +cd src/paddle_ocr && docker compose up -d ocr-cpu + +# 2. Run notebook with papermill +cd src +nohup papermill paddle_ocr_raytune_rest.ipynb output_raytune.ipynb > papermill.log 2>&1 & + +# 3. Monitor +tail -f papermill.log +``` diff --git a/src/output_raytune.ipynb b/src/output_raytune.ipynb new file mode 100644 index 0000000..af5a163 --- /dev/null +++ b/src/output_raytune.ipynb @@ -0,0 +1,2511 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "header", + "metadata": { + "papermill": { + "duration": 0.002078, + "end_time": "2026-01-18T07:09:37.528058", + "exception": false, + "start_time": "2026-01-18T07:09:37.525980", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# PaddleOCR Hyperparameter Optimization via REST API\n", + "\n", + "This notebook runs Ray Tune hyperparameter search calling the PaddleOCR REST API (Docker container).\n", + "\n", + "**Benefits:**\n", + "- No model reload per trial - Model stays loaded in Docker container\n", + "- Faster trials - Skip ~10s model load time per trial\n", + "- Cleaner code - REST API replaces subprocess + CLI arg parsing" + ] + }, + { + "cell_type": "markdown", + "id": "prereq", + "metadata": { + "papermill": { + "duration": 0.000885, + "end_time": "2026-01-18T07:09:37.531707", + "exception": false, + "start_time": "2026-01-18T07:09:37.530822", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Prerequisites\n", + "\n", + "Start 2 PaddleOCR workers for parallel hyperparameter tuning:\n", + "\n", + "```bash\n", + "cd src/paddle_ocr\n", + "docker compose -f docker-compose.workers.yml up\n", + "```\n", + "\n", + "This starts 2 GPU workers on ports 8001-8002, allowing 2 concurrent trials.\n", + "\n", + "For CPU-only systems:\n", + "```bash\n", + "docker compose -f docker-compose.workers.yml --profile cpu up\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "3ob9fsoilc4", + "metadata": { + "papermill": { + "duration": 0.000857, + "end_time": "2026-01-18T07:09:37.533450", + "exception": false, + "start_time": "2026-01-18T07:09:37.532593", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## 0. Dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "wyr2nsoj7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-01-18T07:09:37.536224Z", + "iopub.status.busy": "2026-01-18T07:09:37.536085Z", + "iopub.status.idle": "2026-01-18T07:09:38.812692Z", + "shell.execute_reply": "2026-01-18T07:09:38.811660Z" + }, + "papermill": { + "duration": 1.278901, + "end_time": "2026-01-18T07:09:38.813189", + "exception": false, + "start_time": "2026-01-18T07:09:37.534288", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: ray[tune] in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (2.53.0)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: click>=7.0 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (8.3.1)\r\n", + "Requirement already satisfied: filelock in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (3.20.3)\r\n", + "Requirement already satisfied: jsonschema in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (4.26.0)\r\n", + "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (1.1.2)\r\n", + "Requirement already satisfied: packaging>=24.2 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (25.0)\r\n", + "Requirement already satisfied: protobuf>=3.20.3 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (6.33.4)\r\n", + "Requirement already satisfied: pyyaml in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (6.0.3)\r\n", + "Requirement already satisfied: requests in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (2.32.5)\r\n", + "Requirement already satisfied: pandas in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (2.3.3)\r\n", + "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.10.*,!=2.11.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (2.12.5)\r\n", + "Requirement already satisfied: tensorboardX>=1.9 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (2.6.4)\r\n", + "Requirement already satisfied: pyarrow>=9.0.0 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (22.0.0)\r\n", + "Requirement already satisfied: fsspec in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from ray[tune]) (2026.1.0)\r\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.10.*,!=2.11.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3->ray[tune]) (0.7.0)\r\n", + "Requirement already satisfied: pydantic-core==2.41.5 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.10.*,!=2.11.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3->ray[tune]) (2.41.5)\r\n", + "Requirement already satisfied: typing-extensions>=4.14.1 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.10.*,!=2.11.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3->ray[tune]) (4.15.0)\r\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.10.*,!=2.11.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3->ray[tune]) (0.4.2)\r\n", + "Requirement already satisfied: numpy in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from tensorboardX>=1.9->ray[tune]) (2.4.1)\r\n", + "Requirement already satisfied: attrs>=22.2.0 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from jsonschema->ray[tune]) (25.4.0)\r\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from jsonschema->ray[tune]) (2025.9.1)\r\n", + "Requirement already satisfied: referencing>=0.28.4 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from jsonschema->ray[tune]) (0.37.0)\r\n", + "Requirement already satisfied: rpds-py>=0.25.0 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from jsonschema->ray[tune]) (0.30.0)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: python-dateutil>=2.8.2 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pandas->ray[tune]) (2.9.0.post0)\r\n", + "Requirement already satisfied: pytz>=2020.1 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pandas->ray[tune]) (2025.2)\r\n", + "Requirement already satisfied: tzdata>=2022.7 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pandas->ray[tune]) (2025.3)\r\n", + "Requirement already satisfied: six>=1.5 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas->ray[tune]) (1.17.0)\r\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from requests->ray[tune]) (3.4.4)\r\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from requests->ray[tune]) (3.11)\r\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from requests->ray[tune]) (2.6.3)\r\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from requests->ray[tune]) (2026.1.4)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: optuna in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (4.6.0)\r\n", + "Requirement already satisfied: alembic>=1.5.0 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from optuna) (1.18.1)\r\n", + "Requirement already satisfied: colorlog in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from optuna) (6.10.1)\r\n", + "Requirement already satisfied: numpy in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from optuna) (2.4.1)\r\n", + "Requirement already satisfied: packaging>=20.0 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from optuna) (25.0)\r\n", + "Requirement already satisfied: sqlalchemy>=1.4.2 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from optuna) (2.0.45)\r\n", + "Requirement already satisfied: tqdm in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from optuna) (4.67.1)\r\n", + "Requirement already satisfied: PyYAML in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from optuna) (6.0.3)\r\n", + "Requirement already satisfied: Mako in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from alembic>=1.5.0->optuna) (1.3.10)\r\n", + "Requirement already satisfied: typing-extensions>=4.12 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from alembic>=1.5.0->optuna) (4.15.0)\r\n", + "Requirement already satisfied: greenlet>=1 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from sqlalchemy>=1.4.2->optuna) (3.3.0)\r\n", + "Requirement already satisfied: MarkupSafe>=0.9.2 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from Mako->alembic>=1.5.0->optuna) (3.0.3)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: requests in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (2.32.5)\r\n", + "Requirement already satisfied: pandas in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (2.3.3)\r\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from requests) (3.4.4)\r\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from requests) (3.11)\r\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from requests) (2.6.3)\r\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from requests) (2026.1.4)\r\n", + "Requirement already satisfied: numpy>=1.26.0 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pandas) (2.4.1)\r\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pandas) (2.9.0.post0)\r\n", + "Requirement already satisfied: pytz>=2020.1 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pandas) (2025.2)\r\n", + "Requirement already satisfied: tzdata>=2022.7 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from pandas) (2025.3)\r\n", + "Requirement already satisfied: six>=1.5 in /home/sergio/MastersThesis/.venv/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "# Install dependencies (run once)\n", + "%pip install -U \"ray[tune]\"\n", + "%pip install optuna\n", + "%pip install requests pandas" + ] + }, + { + "cell_type": "markdown", + "id": "imports-header", + "metadata": { + "papermill": { + "duration": 0.009527, + "end_time": "2026-01-18T07:09:38.824238", + "exception": false, + "start_time": "2026-01-18T07:09:38.814711", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## 1. Imports & Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "imports", + "metadata": { + "execution": { + "iopub.execute_input": "2026-01-18T07:09:38.827592Z", + "iopub.status.busy": "2026-01-18T07:09:38.827455Z", + "iopub.status.idle": "2026-01-18T07:09:39.774594Z", + "shell.execute_reply": "2026-01-18T07:09:39.774114Z" + }, + "papermill": { + "duration": 0.949919, + "end_time": "2026-01-18T07:09:39.775317", + "exception": false, + "start_time": "2026-01-18T07:09:38.825398", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "from datetime import datetime\n", + "\n", + "import requests\n", + "import pandas as pd\n", + "\n", + "import ray\n", + "from ray import tune, air\n", + "from ray.tune.search.optuna import OptunaSearch" + ] + }, + { + "cell_type": "markdown", + "id": "config-header", + "metadata": { + "papermill": { + "duration": 0.009564, + "end_time": "2026-01-18T07:09:39.786378", + "exception": false, + "start_time": "2026-01-18T07:09:39.776814", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## 2. API Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "config", + "metadata": { + "execution": { + "iopub.execute_input": "2026-01-18T07:09:39.789662Z", + "iopub.status.busy": "2026-01-18T07:09:39.789533Z", + "iopub.status.idle": "2026-01-18T07:09:39.791831Z", + "shell.execute_reply": "2026-01-18T07:09:39.791432Z" + }, + "papermill": { + "duration": 0.004949, + "end_time": "2026-01-18T07:09:39.792449", + "exception": false, + "start_time": "2026-01-18T07:09:39.787500", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# PaddleOCR REST API endpoints - 2 workers for parallel trials\n", + "# Start workers with: cd src/paddle_ocr && docker compose -f docker-compose.workers.yml up\n", + "WORKER_PORTS = [8001, 8002]\n", + "WORKER_URLS = [f\"http://localhost:{port}\" for port in WORKER_PORTS]\n", + "\n", + "# Output folder for results\n", + "OUTPUT_FOLDER = \"results\"\n", + "os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n", + "\n", + "# Number of concurrent trials = number of workers\n", + "NUM_WORKERS = len(WORKER_URLS)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "health-check", + "metadata": { + "execution": { + "iopub.execute_input": "2026-01-18T07:09:39.795399Z", + "iopub.status.busy": "2026-01-18T07:09:39.795279Z", + "iopub.status.idle": "2026-01-18T07:09:39.803656Z", + "shell.execute_reply": "2026-01-18T07:09:39.803234Z" + }, + "papermill": { + "duration": 0.010311, + "end_time": "2026-01-18T07:09:39.803886", + "exception": false, + "start_time": "2026-01-18T07:09:39.793575", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✓ http://localhost:8001: ok (GPU: None)\n", + "✓ http://localhost:8002: ok (GPU: None)\n", + "\n", + "2/2 workers ready for parallel tuning\n" + ] + } + ], + "source": [ + "# Verify all workers are running\n", + "healthy_workers = []\n", + "for url in WORKER_URLS:\n", + " try:\n", + " health = requests.get(f\"{url}/health\", timeout=10).json()\n", + " if health['status'] == 'ok' and health['model_loaded']:\n", + " healthy_workers.append(url)\n", + " print(f\"✓ {url}: {health['status']} (GPU: {health.get('gpu_name', 'N/A')})\")\n", + " else:\n", + " print(f\"✗ {url}: not ready yet\")\n", + " except requests.exceptions.ConnectionError:\n", + " print(f\"✗ {url}: not reachable\")\n", + "\n", + "if not healthy_workers:\n", + " raise RuntimeError(\n", + " \"No healthy workers found. Start them with:\\n\"\n", + " \" cd src/paddle_ocr && docker compose -f docker-compose.workers.yml up\"\n", + " )\n", + "\n", + "print(f\"\\n{len(healthy_workers)}/{len(WORKER_URLS)} workers ready for parallel tuning\")" + ] + }, + { + "cell_type": "markdown", + "id": "search-space-header", + "metadata": { + "papermill": { + "duration": 0.001081, + "end_time": "2026-01-18T07:09:39.806136", + "exception": false, + "start_time": "2026-01-18T07:09:39.805055", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## 3. Search Space" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "search-space", + "metadata": { + "execution": { + "iopub.execute_input": "2026-01-18T07:09:39.808859Z", + "iopub.status.busy": "2026-01-18T07:09:39.808794Z", + "iopub.status.idle": "2026-01-18T07:09:39.810819Z", + "shell.execute_reply": "2026-01-18T07:09:39.810510Z" + }, + "papermill": { + "duration": 0.003948, + "end_time": "2026-01-18T07:09:39.811160", + "exception": false, + "start_time": "2026-01-18T07:09:39.807212", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "search_space = {\n", + " # Whether to use document image orientation classification\n", + " \"use_doc_orientation_classify\": tune.choice([True, False]),\n", + " # Whether to use text image unwarping\n", + " \"use_doc_unwarping\": tune.choice([True, False]),\n", + " # Whether to use text line orientation classification\n", + " \"textline_orientation\": tune.choice([True, False]),\n", + " # Detection pixel threshold (pixels > threshold are considered text)\n", + " \"text_det_thresh\": tune.uniform(0.0, 0.7),\n", + " # Detection box threshold (average score within border)\n", + " \"text_det_box_thresh\": tune.uniform(0.0, 0.7),\n", + " # Text detection expansion coefficient\n", + " \"text_det_unclip_ratio\": tune.choice([0.0]),\n", + " # Text recognition threshold (filter low confidence results)\n", + " \"text_rec_score_thresh\": tune.uniform(0.0, 0.7),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "trainable-header", + "metadata": { + "papermill": { + "duration": 0.001066, + "end_time": "2026-01-18T07:09:39.813352", + "exception": false, + "start_time": "2026-01-18T07:09:39.812286", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## 4. Trainable Function" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "trainable", + "metadata": { + "execution": { + "iopub.execute_input": "2026-01-18T07:09:39.815935Z", + "iopub.status.busy": "2026-01-18T07:09:39.815875Z", + "iopub.status.idle": "2026-01-18T07:09:39.819033Z", + "shell.execute_reply": "2026-01-18T07:09:39.818672Z" + }, + "papermill": { + "duration": 0.00507, + "end_time": "2026-01-18T07:09:39.819491", + "exception": false, + "start_time": "2026-01-18T07:09:39.814421", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def trainable_paddle_ocr(config):\n", + " \"\"\"Call PaddleOCR REST API with the given hyperparameter config.\n", + " \n", + " Uses trial index to deterministically assign a worker (round-robin),\n", + " ensuring only 1 request per container at a time.\n", + " \"\"\"\n", + " import requests # Must be inside function for Ray workers\n", + " from ray import train\n", + "\n", + " # Worker URLs - round-robin assignment based on trial index\n", + " WORKER_PORTS = [8001, 8002]\n", + " NUM_WORKERS = len(WORKER_PORTS)\n", + " \n", + " # Get trial context for deterministic worker assignment\n", + " context = train.get_context()\n", + " trial_id = context.get_trial_id() if context else \"0\"\n", + " # Extract numeric part from trial ID (e.g., \"trainable_paddle_ocr_abc123_00001\" -> 1)\n", + " try:\n", + " trial_num = int(trial_id.split(\"_\")[-1])\n", + " except (ValueError, IndexError):\n", + " trial_num = hash(trial_id)\n", + " \n", + " worker_idx = trial_num % NUM_WORKERS\n", + " api_url = f\"http://localhost:{WORKER_PORTS[worker_idx]}\"\n", + "\n", + " payload = {\n", + " \"pdf_folder\": \"/app/dataset\",\n", + " \"use_doc_orientation_classify\": config.get(\"use_doc_orientation_classify\", False),\n", + " \"use_doc_unwarping\": config.get(\"use_doc_unwarping\", False),\n", + " \"textline_orientation\": config.get(\"textline_orientation\", True),\n", + " \"text_det_thresh\": config.get(\"text_det_thresh\", 0.0),\n", + " \"text_det_box_thresh\": config.get(\"text_det_box_thresh\", 0.0),\n", + " \"text_det_unclip_ratio\": config.get(\"text_det_unclip_ratio\", 1.5),\n", + " \"text_rec_score_thresh\": config.get(\"text_rec_score_thresh\", 0.0),\n", + " \"start_page\": 5,\n", + " \"end_page\": 10,\n", + " }\n", + "\n", + " try:\n", + " response = requests.post(f\"{api_url}/evaluate\", json=payload, timeout=600)\n", + " response.raise_for_status()\n", + " metrics = response.json()\n", + " metrics[\"worker\"] = api_url\n", + " train.report(metrics)\n", + " except Exception as e:\n", + " train.report({\n", + " \"CER\": 1.0,\n", + " \"WER\": 1.0,\n", + " \"TIME\": 0.0,\n", + " \"PAGES\": 0,\n", + " \"TIME_PER_PAGE\": 0,\n", + " \"worker\": api_url,\n", + " \"ERROR\": str(e)[:500]\n", + " })" + ] + }, + { + "cell_type": "markdown", + "id": "tuner-header", + "metadata": { + "papermill": { + "duration": 0.001049, + "end_time": "2026-01-18T07:09:39.821656", + "exception": false, + "start_time": "2026-01-18T07:09:39.820607", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## 5. Run Tuner" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ray-init", + "metadata": { + "execution": { + "iopub.execute_input": "2026-01-18T07:09:39.824409Z", + "iopub.status.busy": "2026-01-18T07:09:39.824357Z", + "iopub.status.idle": "2026-01-18T07:09:42.432539Z", + "shell.execute_reply": "2026-01-18T07:09:42.431817Z" + }, + "papermill": { + "duration": 2.610279, + "end_time": "2026-01-18T07:09:42.433000", + "exception": false, + "start_time": "2026-01-18T07:09:39.822721", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:41,083\tINFO worker.py:2007 -- Started a local Ray instance.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ray Tune ready (version: 2.53.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py:2046: FutureWarning: Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "ray.init(ignore_reinit_error=True)\n", + "print(f\"Ray Tune ready (version: {ray.__version__})\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "tuner", + "metadata": { + "execution": { + "iopub.execute_input": "2026-01-18T07:09:42.446046Z", + "iopub.status.busy": "2026-01-18T07:09:42.445323Z" + }, + "papermill": { + "duration": null, + "end_time": null, + "exception": false, + "start_time": "2026-01-18T07:09:42.434682", + "status": "running" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/impl/tuner_internal.py:144: RayDeprecationWarning: The `RunConfig` class should be imported from `ray.tune` when passing it to the Tuner. Please update your imports. See this issue for more context and migration options: https://github.com/ray-project/ray/issues/49454. Disable these warnings by setting the environment variable: RAY_TRAIN_ENABLE_V2_MIGRATION_WARNINGS=0\n", + " _log_deprecation_warning(\n", + "2026-01-18 08:09:42,457\tINFO tune.py:616 -- [output] This uses the legacy output and progress reporter, as Jupyter notebooks are not supported by the new engine, yet. For more information, please see https://github.com/ray-project/ray/issues/36949\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2026-01-18 08:09:42,461] A new study created in memory with name: optuna\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "
\n", + "
\n", + "

Tune Status

\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Current time:2026-01-18 08:10:43
Running for: 00:01:00.59
Memory: 11.1/119.7 GiB
\n", + "
\n", + "
\n", + "
\n", + "

System Info

\n", + " Using FIFO scheduling algorithm.
Logical resource usage: 1.0/20 CPUs, 0/1 GPUs (0.0/1.0 accelerator_type:GB10)\n", + "
\n", + "
\n", + "
\n", + "

Messages

\n", + " \n", + " ... 17 more trials not shown (17 ERROR)\n", + " Number of errored trials: 36
Table truncated to 20 rows (16 overflow)
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Trial name # failureserror file
trainable_paddle_ocr_1dfb6efc 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_1dfb6efc_1_text_det_box_thresh=0.3434,text_det_thresh=0.5450,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_08-09-42/error.txt
trainable_paddle_ocr_7207923f 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_7207923f_2_text_det_box_thresh=0.5582,text_det_thresh=0.5795,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_08-09-44/error.txt
trainable_paddle_ocr_9d186472 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_9d186472_3_text_det_box_thresh=0.6292,text_det_thresh=0.5747,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_08-09-45/error.txt
trainable_paddle_ocr_cf1ca69b 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_cf1ca69b_4_text_det_box_thresh=0.5197,text_det_thresh=0.5634,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_08-09-47/error.txt
trainable_paddle_ocr_8705afb0 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_8705afb0_5_text_det_box_thresh=0.2333,text_det_thresh=0.4399,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_08-09-49/error.txt
trainable_paddle_ocr_6a87c44e 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_6a87c44e_6_text_det_box_thresh=0.6925,text_det_thresh=0.1092,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_08-09-50/error.txt
trainable_paddle_ocr_b234b9c4 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_b234b9c4_7_text_det_box_thresh=0.0388,text_det_thresh=0.1447,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_08-09-52/error.txt
trainable_paddle_ocr_0c097d1b 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_0c097d1b_8_text_det_box_thresh=0.3627,text_det_thresh=0.5081,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_08-09-54/error.txt
trainable_paddle_ocr_65aef77d 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_65aef77d_9_text_det_box_thresh=0.5284,text_det_thresh=0.0341,text_det_unclip_ratio=0.0000,text_rec_score_thre_2026-01-18_08-09-55/error.txt
trainable_paddle_ocr_1bdb346f 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_1bdb346f_10_text_det_box_thresh=0.2700,text_det_thresh=0.0133,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-09-57/error.txt
trainable_paddle_ocr_9eb43564 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_9eb43564_11_text_det_box_thresh=0.0667,text_det_thresh=0.6018,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-09-59/error.txt
trainable_paddle_ocr_69102b85 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_69102b85_12_text_det_box_thresh=0.0186,text_det_thresh=0.1448,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-10-00/error.txt
trainable_paddle_ocr_2393de4f 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_2393de4f_13_text_det_box_thresh=0.2169,text_det_thresh=0.1937,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-10-02/error.txt
trainable_paddle_ocr_e3fa0b1b 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_e3fa0b1b_14_text_det_box_thresh=0.2970,text_det_thresh=0.0638,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-10-04/error.txt
trainable_paddle_ocr_1cf2c62c 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_1cf2c62c_15_text_det_box_thresh=0.2257,text_det_thresh=0.3085,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-10-05/error.txt
trainable_paddle_ocr_9e723005 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_9e723005_16_text_det_box_thresh=0.5469,text_det_thresh=0.3089,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-10-07/error.txt
trainable_paddle_ocr_9751b8b7 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_9751b8b7_17_text_det_box_thresh=0.6950,text_det_thresh=0.5605,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-10-09/error.txt
trainable_paddle_ocr_20eeca6a 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_20eeca6a_18_text_det_box_thresh=0.2342,text_det_thresh=0.4196,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-10-10/error.txt
trainable_paddle_ocr_336cd7e0 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_336cd7e0_19_text_det_box_thresh=0.4674,text_det_thresh=0.2587,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-10-12/error.txt
trainable_paddle_ocr_3f7fde37 1/tmp/ray/session_2026-01-18_08-09-39_842185_1162060/artifacts/2026-01-18_08-09-42/trainable_paddle_ocr_2026-01-18_08-09-42/driver_artifacts/trainable_paddle_ocr_3f7fde37_20_text_det_box_thresh=0.4702,text_det_thresh=0.5269,text_det_unclip_ratio=0.0000,text_rec_score_thr_2026-01-18_08-10-14/error.txt
\n", + "
\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Trial Status

\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Trial name status loc text_det_box_thresh text_det_thresh text_det_unclip_rati\n", + "o text_rec_score_thres\n", + "htextline_orientation use_doc_orientation_\n", + "classify use_doc_unwarping
trainable_paddle_ocr_43946461PENDING 0.629564 0.682338 00.110796 True FalseTrue
trainable_paddle_ocr_0c097d1bERROR 192.168.65.140:1164070 0.362666 0.508107 00.189003 True FalseTrue
trainable_paddle_ocr_1826c000ERROR 192.168.65.140:1165715 0.57287 0.203366 00.125037 False True False
trainable_paddle_ocr_18c164caERROR 192.168.65.140:1166093 0.63944 0.381276 00.56397 False FalseFalse
trainable_paddle_ocr_1bdb346fERROR 192.168.65.140:1164216 0.269986 0.013320900.4966 True FalseFalse
trainable_paddle_ocr_1cf2c62cERROR 192.168.65.140:1164584 0.225659 0.308462 00.181031 False FalseFalse
trainable_paddle_ocr_1dfb6efcERROR 192.168.65.140:1163551 0.3434 0.545037 00.263803 True True True
trainable_paddle_ocr_20eeca6aERROR 192.168.65.140:1164817 0.234161 0.419637 00.270497 True True False
trainable_paddle_ocr_2393de4fERROR 192.168.65.140:1164465 0.216897 0.193734 00.170636 True FalseFalse
trainable_paddle_ocr_332b7991ERROR 192.168.65.140:1165567 0.241403 0.038812700.414523 False FalseFalse
trainable_paddle_ocr_336cd7e0ERROR 192.168.65.140:1164898 0.467436 0.258686 00.138666 True True False
trainable_paddle_ocr_39036a64ERROR 192.168.65.140:1165427 0.570841 0.479926 00.272159 False FalseFalse
trainable_paddle_ocr_395f1e8bERROR 192.168.65.140:1165855 0.363105 0.039043800.34921 True True False
trainable_paddle_ocr_3f7fde37ERROR 192.168.65.140:1164963 0.470214 0.526887 00.0699669False FalseTrue
trainable_paddle_ocr_65aef77dERROR 192.168.65.140:1164138 0.528446 0.034088800.0928365False FalseTrue
trainable_paddle_ocr_69102b85ERROR 192.168.65.140:1164361 0.0186252 0.144809 00.0397942True True True
trainable_paddle_ocr_69575fc2ERROR 192.168.65.140:1165304 0.495832 0.237163 00.653164 False True True
trainable_paddle_ocr_6a87c44eERROR 192.168.65.140:1163924 0.692497 0.1092 00.55452 True True True
trainable_paddle_ocr_7207923fERROR 192.168.65.140:1163640 0.558151 0.579507 00.301832 False FalseTrue
trainable_paddle_ocr_77b79aebERROR 192.168.65.140:1165797 0.373257 0.694542 00.0908941False True False
\n", + "
\n", + "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:44,254\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_1dfb6efc\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1163551, ip=192.168.65.140, actor_id=fca623a6d8fd15f5f6b7f9d601000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "

Trial Progress

\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Trial name
trainable_paddle_ocr_0c097d1b
trainable_paddle_ocr_1826c000
trainable_paddle_ocr_18c164ca
trainable_paddle_ocr_1bdb346f
trainable_paddle_ocr_1cf2c62c
trainable_paddle_ocr_1dfb6efc
trainable_paddle_ocr_20eeca6a
trainable_paddle_ocr_2393de4f
trainable_paddle_ocr_332b7991
trainable_paddle_ocr_336cd7e0
trainable_paddle_ocr_39036a64
trainable_paddle_ocr_395f1e8b
trainable_paddle_ocr_3f7fde37
trainable_paddle_ocr_65aef77d
trainable_paddle_ocr_69102b85
trainable_paddle_ocr_69575fc2
trainable_paddle_ocr_6a87c44e
trainable_paddle_ocr_7207923f
trainable_paddle_ocr_77b79aeb
trainable_paddle_ocr_7caf7679
trainable_paddle_ocr_826a3bfd
trainable_paddle_ocr_83ae14a3
trainable_paddle_ocr_8705afb0
trainable_paddle_ocr_9751b8b7
trainable_paddle_ocr_9d186472
trainable_paddle_ocr_9e723005
trainable_paddle_ocr_9eb43564
trainable_paddle_ocr_a369d771
trainable_paddle_ocr_a74cc67d
trainable_paddle_ocr_ace50f2b
trainable_paddle_ocr_b234b9c4
trainable_paddle_ocr_b9a9db46
trainable_paddle_ocr_bd3b2d27
trainable_paddle_ocr_cf1ca69b
trainable_paddle_ocr_e3fa0b1b
trainable_paddle_ocr_f4cf9600
\n", + "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:45,939\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_7207923f\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1163640, ip=192.168.65.140, actor_id=111e7b2e86b54e45900ebbf401000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:47,616\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_9d186472\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1163704, ip=192.168.65.140, actor_id=de9e40c2993f255a035dc29801000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:49,285\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_cf1ca69b\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1163806, ip=192.168.65.140, actor_id=7aa6aff7d79803e2f6a0ea8801000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:50,940\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_8705afb0\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1163865, ip=192.168.65.140, actor_id=8d83a5d435aa059c2d8a593001000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:52,601\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_6a87c44e\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1163924, ip=192.168.65.140, actor_id=9b88e959bc0aa616a7f01c1a01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:54,239\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_b234b9c4\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164002, ip=192.168.65.140, actor_id=684fd5c585af1b5fccf74f6301000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:55,885\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_0c097d1b\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164070, ip=192.168.65.140, actor_id=d146676e152867f73062234d01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:57,552\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_65aef77d\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164138, ip=192.168.65.140, actor_id=7ae43cf36b1f62812187a8c701000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:09:59,220\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_1bdb346f\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164216, ip=192.168.65.140, actor_id=229efb5639262d4e4c6553fb01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:00,884\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_9eb43564\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164304, ip=192.168.65.140, actor_id=afba9408b304d3fb53fa0be901000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:02,576\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_69102b85\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164361, ip=192.168.65.140, actor_id=839891072c4e9b45332f821f01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:04,238\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_2393de4f\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164465, ip=192.168.65.140, actor_id=a1fdaeb2cb395b787b9e94bb01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:05,907\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_e3fa0b1b\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164525, ip=192.168.65.140, actor_id=7dcea88a70a5d6d062105d1c01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:07,583\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_1cf2c62c\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164584, ip=192.168.65.140, actor_id=e677471edcdc3fa119c2825401000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:09,250\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_9e723005\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164664, ip=192.168.65.140, actor_id=f2e25366365217e9dc91f57201000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(pid=gcs_server)\u001b[0m [2026-01-18 08:10:09,972 E 1162212 1162212] (gcs_server) gcs_server.cc:303: Failed to establish connection to the event+metrics exporter agent. Events and metrics will not be exported. Exporter agent status: RpcError: Running out of retries to initialize the metrics agent. rpc_code: 14\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:10,918\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_9751b8b7\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164723, ip=192.168.65.140, actor_id=d2491bdf054d8db1e98798f701000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[33m(raylet)\u001b[0m [2026-01-18 08:10:11,011 E 1162379 1162379] (raylet) main.cc:1032: Failed to establish connection to the metrics exporter agent. Metrics will not be exported. Exporter agent status: RpcError: Running out of retries to initialize the metrics agent. rpc_code: 14\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(bundle_reservation_check_func pid=1162459)\u001b[0m [2026-01-18 08:10:11,819 E 1162459 1162599] core_worker_process.cc:842: Failed to establish connection to the metrics exporter agent. Metrics will not be exported. Exporter agent status: RpcError: Running out of retries to initialize the metrics agent. rpc_code: 14\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2026-01-18 08:10:12,432 E 1162060 1162448] core_worker_process.cc:842: Failed to establish connection to the metrics exporter agent. Metrics will not be exported. Exporter agent status: RpcError: Running out of retries to initialize the metrics agent. rpc_code: 14\n", + "2026-01-18 08:10:12,581\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_20eeca6a\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164817, ip=192.168.65.140, actor_id=4bcb09cff39a0059fff3b84f01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:14,246\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_336cd7e0\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164898, ip=192.168.65.140, actor_id=3a19705a167b69a89bb77fad01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:15,922\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_3f7fde37\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1164963, ip=192.168.65.140, actor_id=fe86fb13d5683fe32e12d62301000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:17,609\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_a74cc67d\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165021, ip=192.168.65.140, actor_id=9f3d47f5faeb28a9b06cdb7801000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:19,251\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_bd3b2d27\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165105, ip=192.168.65.140, actor_id=578a91cf6da9a32d9266a07601000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:20,907\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_ace50f2b\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165165, ip=192.168.65.140, actor_id=57b527fb486c439e0da549a201000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:22,567\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_f4cf9600\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165224, ip=192.168.65.140, actor_id=4b4d142f4b73290da4296a5901000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:24,216\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_69575fc2\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165304, ip=192.168.65.140, actor_id=c7cec2a8e27b97dd3a21c7dd01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:25,885\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_83ae14a3\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165368, ip=192.168.65.140, actor_id=0b1a528e4b4d67fc12c398e901000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:27,550\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_39036a64\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165427, ip=192.168.65.140, actor_id=346b0c0514cf182156c9351001000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:29,194\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_7caf7679\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165507, ip=192.168.65.140, actor_id=ae46187a690c35ca9c9d808201000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:30,870\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_332b7991\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165567, ip=192.168.65.140, actor_id=c7e6e565cd25486d8ee1ce8901000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:32,514\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_b9a9db46\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165624, ip=192.168.65.140, actor_id=cf4810f5df446624f67c8e9f01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:34,190\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_1826c000\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165715, ip=192.168.65.140, actor_id=4a18dbfb5d9f6f3b24b8e44001000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:35,864\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_77b79aeb\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165797, ip=192.168.65.140, actor_id=e069e7254b4bdec785604bc901000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:37,558\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_395f1e8b\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165855, ip=192.168.65.140, actor_id=126e55037174bf919a34778401000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:39,230\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_a369d771\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165918, ip=192.168.65.140, actor_id=20667fba64228d7cf4ae365d01000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:40,868\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_826a3bfd\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1165977, ip=192.168.65.140, actor_id=8815169d89c0f8b8560cbbc801000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-18 08:10:42,550\tERROR tune_controller.py:1331 -- Trial task failed for trial trainable_paddle_ocr_18c164ca\n", + "Traceback (most recent call last):\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(DeprecationWarning): \u001b[36mray::ImplicitFunc.train()\u001b[39m (pid=1166093, ip=192.168.65.140, actor_id=03943e8081fc8cd326f6372201000000, repr=trainable_paddle_ocr)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/air/_internal/util.py\", line 98, in run\n", + " self._ret = self._target(*self._args, **self._kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 44, in \n", + " training_func=lambda: self._trainable_func(self.config),\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py\", line 249, in _trainable_func\n", + " output = fn()\n", + " ^^^^\n", + " File \"/tmp/ipykernel_1162060/2443294666.py\", line 15, in trainable_paddle_ocr\n", + " File \"/home/sergio/MastersThesis/.venv/lib/python3.12/site-packages/ray/train/v2/_internal/util.py\", line 273, in _wrapped_fn\n", + " raise DeprecationWarning(\n", + "DeprecationWarning: `ray.train.get_context` is deprecated when running in a function passed to Ray Tune. Please use the equivalent `ray.tune` API instead. See this issue for more context: https://github.com/ray-project/ray/issues/49454\n" + ] + } + ], + "source": [ + "tuner = tune.Tuner(\n", + " trainable_paddle_ocr,\n", + " tune_config=tune.TuneConfig(\n", + " metric=\"CER\",\n", + " mode=\"min\",\n", + " search_alg=OptunaSearch(),\n", + " num_samples=64,\n", + " max_concurrent_trials=NUM_WORKERS, # Run trials in parallel across workers\n", + " ),\n", + " run_config=air.RunConfig(verbose=2, log_to_file=False),\n", + " param_space=search_space,\n", + ")\n", + "\n", + "results = tuner.fit()" + ] + }, + { + "cell_type": "markdown", + "id": "analysis-header", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "pending" + }, + "tags": [] + }, + "source": [ + "## 6. Results Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "results-df", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "pending" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "df = results.get_dataframe()\n", + "df.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "save-results", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "pending" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Save results to CSV\n", + "timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n", + "filename = f\"raytune_paddle_rest_results_{timestamp}.csv\"\n", + "filepath = os.path.join(OUTPUT_FOLDER, filename)\n", + "\n", + "df.to_csv(filepath, index=False)\n", + "print(f\"Results saved: {filepath}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "best-config", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "pending" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Best configuration\n", + "best = df.loc[df[\"CER\"].idxmin()]\n", + "\n", + "print(f\"Best CER: {best['CER']:.6f}\")\n", + "print(f\"Best WER: {best['WER']:.6f}\")\n", + "print(f\"\\nOptimal Configuration:\")\n", + "print(f\" textline_orientation: {best['config/textline_orientation']}\")\n", + "print(f\" use_doc_orientation_classify: {best['config/use_doc_orientation_classify']}\")\n", + "print(f\" use_doc_unwarping: {best['config/use_doc_unwarping']}\")\n", + "print(f\" text_det_thresh: {best['config/text_det_thresh']:.4f}\")\n", + "print(f\" text_det_box_thresh: {best['config/text_det_box_thresh']:.4f}\")\n", + "print(f\" text_det_unclip_ratio: {best['config/text_det_unclip_ratio']}\")\n", + "print(f\" text_rec_score_thresh: {best['config/text_rec_score_thresh']:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "correlation", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "pending" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Correlation analysis\n", + "param_cols = [\n", + " \"config/text_det_thresh\",\n", + " \"config/text_det_box_thresh\",\n", + " \"config/text_det_unclip_ratio\",\n", + " \"config/text_rec_score_thresh\",\n", + "]\n", + "\n", + "corr_cer = df[param_cols + [\"CER\"]].corr()[\"CER\"].sort_values(ascending=False)\n", + "corr_wer = df[param_cols + [\"WER\"]].corr()[\"WER\"].sort_values(ascending=False)\n", + "\n", + "print(\"Correlation with CER:\")\n", + "print(corr_cer)\n", + "print(\"\\nCorrelation with WER:\")\n", + "print(corr_wer)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + }, + "papermill": { + "default_parameters": {}, + "duration": null, + "end_time": null, + "environment_variables": {}, + "exception": null, + "input_path": "paddle_ocr_raytune_rest.ipynb", + "output_path": "output_raytune.ipynb", + "parameters": {}, + "start_time": "2026-01-18T07:09:36.905858", + "version": "2.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/paddle_ocr/README.md b/src/paddle_ocr/README.md index 26fa2f9..ee2cca6 100644 --- a/src/paddle_ocr/README.md +++ b/src/paddle_ocr/README.md @@ -520,6 +520,28 @@ docker load < paddle-ocr-arm64.tar.gz ## Using with Ray Tune +### Multi-Worker Setup for Parallel Trials + +Run multiple workers for parallel hyperparameter tuning: + +```bash +cd src/paddle_ocr + +# Start 2 CPU workers (ports 8001-8002) +sudo docker compose -f docker-compose.workers.yml --profile cpu up -d + +# Or for GPU workers (if supported) +sudo docker compose -f docker-compose.workers.yml --profile gpu up -d + +# Check workers are healthy +curl http://localhost:8001/health +curl http://localhost:8002/health +``` + +Then run the notebook with `max_concurrent_trials=2` to use both workers in parallel. + +### Single Worker Setup + Update your notebook's `trainable_paddle_ocr` function: ```python diff --git a/src/paddle_ocr/docker-compose.workers.yml b/src/paddle_ocr/docker-compose.workers.yml new file mode 100644 index 0000000..cc8edde --- /dev/null +++ b/src/paddle_ocr/docker-compose.workers.yml @@ -0,0 +1,90 @@ +# docker-compose.workers.yml - Multiple PaddleOCR workers for parallel Ray Tune +# +# Usage: +# GPU (4 workers sharing GPU): +# docker compose -f docker-compose.workers.yml up +# +# CPU (4 workers): +# docker compose -f docker-compose.workers.yml --profile cpu up +# +# Scale workers (e.g., 8 workers): +# NUM_WORKERS=8 docker compose -f docker-compose.workers.yml up +# +# Each worker runs on a separate port: 8001, 8002, 8003, 8004, ... + +x-ocr-gpu-common: &ocr-gpu-common + image: seryus.ddns.net/unir/paddle-ocr-gpu:latest + volumes: + - ../dataset:/app/dataset:ro + - paddlex-cache:/root/.paddlex + environment: + - PYTHONUNBUFFERED=1 + - CUDA_VISIBLE_DEVICES=0 + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s + +x-ocr-cpu-common: &ocr-cpu-common + image: paddle-ocr-api:cpu + volumes: + - ../dataset:/app/dataset:ro + - paddlex-cache:/root/.paddlex + environment: + - PYTHONUNBUFFERED=1 + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s + +services: + # GPU Workers (gpu profile) - share single GPU + ocr-worker-1: + <<: *ocr-gpu-common + container_name: paddle-ocr-worker-1 + ports: + - "8001:8000" + profiles: + - gpu + + ocr-worker-2: + <<: *ocr-gpu-common + container_name: paddle-ocr-worker-2 + ports: + - "8002:8000" + profiles: + - gpu + + # CPU Workers (cpu profile) - for systems without GPU + ocr-cpu-worker-1: + <<: *ocr-cpu-common + container_name: paddle-ocr-cpu-worker-1 + ports: + - "8001:8000" + profiles: + - cpu + + ocr-cpu-worker-2: + <<: *ocr-cpu-common + container_name: paddle-ocr-cpu-worker-2 + ports: + - "8002:8000" + profiles: + - cpu + +volumes: + paddlex-cache: + name: paddlex-model-cache diff --git a/src/paddle_ocr/paddle_ocr_tuning_rest.py b/src/paddle_ocr/paddle_ocr_tuning_rest.py index f345aba..6e836c6 100644 --- a/src/paddle_ocr/paddle_ocr_tuning_rest.py +++ b/src/paddle_ocr/paddle_ocr_tuning_rest.py @@ -5,6 +5,7 @@ import os import re import time +import threading from typing import Optional from contextlib import asynccontextmanager @@ -61,6 +62,10 @@ class AppState: dataset_path: Optional[str] = None det_model: str = DEFAULT_DET_MODEL rec_model: str = DEFAULT_REC_MODEL + lock: threading.Lock = None # Protects OCR model from concurrent access + + def __init__(self): + self.lock = threading.Lock() state = AppState() @@ -281,28 +286,30 @@ def evaluate(request: EvaluateRequest): time_per_page_list = [] t0 = time.time() - for idx in range(start, end): - img, ref = state.dataset[idx] - arr = np.array(img) + # Lock to prevent concurrent OCR access (model is not thread-safe) + with state.lock: + for idx in range(start, end): + img, ref = state.dataset[idx] + arr = np.array(img) - tp0 = time.time() - out = state.ocr.predict( - arr, - use_doc_orientation_classify=request.use_doc_orientation_classify, - use_doc_unwarping=request.use_doc_unwarping, - use_textline_orientation=request.textline_orientation, - text_det_thresh=request.text_det_thresh, - text_det_box_thresh=request.text_det_box_thresh, - text_det_unclip_ratio=request.text_det_unclip_ratio, - text_rec_score_thresh=request.text_rec_score_thresh, - ) + tp0 = time.time() + out = state.ocr.predict( + arr, + use_doc_orientation_classify=request.use_doc_orientation_classify, + use_doc_unwarping=request.use_doc_unwarping, + use_textline_orientation=request.textline_orientation, + text_det_thresh=request.text_det_thresh, + text_det_box_thresh=request.text_det_box_thresh, + text_det_unclip_ratio=request.text_det_unclip_ratio, + text_rec_score_thresh=request.text_rec_score_thresh, + ) - pred = assemble_from_paddle_result(out) - time_per_page_list.append(float(time.time() - tp0)) + pred = assemble_from_paddle_result(out) + time_per_page_list.append(float(time.time() - tp0)) - m = evaluate_text(ref, pred) - cer_list.append(m["CER"]) - wer_list.append(m["WER"]) + m = evaluate_text(ref, pred) + cer_list.append(m["CER"]) + wer_list.append(m["WER"]) return EvaluateResponse( CER=float(np.mean(cer_list)) if cer_list else 1.0, diff --git a/src/paddle_ocr_raytune_rest.ipynb b/src/paddle_ocr_raytune_rest.ipynb new file mode 100644 index 0000000..9df88f5 --- /dev/null +++ b/src/paddle_ocr_raytune_rest.ipynb @@ -0,0 +1,393 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "header", + "metadata": {}, + "source": [ + "# PaddleOCR Hyperparameter Optimization via REST API\n", + "\n", + "This notebook runs Ray Tune hyperparameter search calling the PaddleOCR REST API (Docker container).\n", + "\n", + "**Benefits:**\n", + "- No model reload per trial - Model stays loaded in Docker container\n", + "- Faster trials - Skip ~10s model load time per trial\n", + "- Cleaner code - REST API replaces subprocess + CLI arg parsing" + ] + }, + { + "cell_type": "markdown", + "id": "prereq", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "Start 2 PaddleOCR workers for parallel hyperparameter tuning:\n", + "\n", + "```bash\n", + "cd src/paddle_ocr\n", + "docker compose -f docker-compose.workers.yml up\n", + "```\n", + "\n", + "This starts 2 GPU workers on ports 8001-8002, allowing 2 concurrent trials.\n", + "\n", + "For CPU-only systems:\n", + "```bash\n", + "docker compose -f docker-compose.workers.yml --profile cpu up\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "3ob9fsoilc4", + "metadata": {}, + "source": [ + "## 0. Dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "wyr2nsoj7", + "metadata": {}, + "outputs": [], + "source": [ + "# Install dependencies (run once)\n", + "%pip install -U \"ray[tune]\"\n", + "%pip install optuna\n", + "%pip install requests pandas" + ] + }, + { + "cell_type": "markdown", + "id": "imports-header", + "metadata": {}, + "source": [ + "## 1. Imports & Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "imports", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from datetime import datetime\n", + "\n", + "import requests\n", + "import pandas as pd\n", + "\n", + "import ray\n", + "from ray import tune, air\n", + "from ray.tune.search.optuna import OptunaSearch" + ] + }, + { + "cell_type": "markdown", + "id": "config-header", + "metadata": {}, + "source": [ + "## 2. API Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "config", + "metadata": {}, + "outputs": [], + "source": [ + "# PaddleOCR REST API endpoints - 2 workers for parallel trials\n", + "# Start workers with: cd src/paddle_ocr && docker compose -f docker-compose.workers.yml up\n", + "WORKER_PORTS = [8001, 8002]\n", + "WORKER_URLS = [f\"http://localhost:{port}\" for port in WORKER_PORTS]\n", + "\n", + "# Output folder for results\n", + "OUTPUT_FOLDER = \"results\"\n", + "os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n", + "\n", + "# Number of concurrent trials = number of workers\n", + "NUM_WORKERS = len(WORKER_URLS)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "health-check", + "metadata": {}, + "outputs": [], + "source": [ + "# Verify all workers are running\n", + "healthy_workers = []\n", + "for url in WORKER_URLS:\n", + " try:\n", + " health = requests.get(f\"{url}/health\", timeout=10).json()\n", + " if health['status'] == 'ok' and health['model_loaded']:\n", + " healthy_workers.append(url)\n", + " print(f\"✓ {url}: {health['status']} (GPU: {health.get('gpu_name', 'N/A')})\")\n", + " else:\n", + " print(f\"✗ {url}: not ready yet\")\n", + " except requests.exceptions.ConnectionError:\n", + " print(f\"✗ {url}: not reachable\")\n", + "\n", + "if not healthy_workers:\n", + " raise RuntimeError(\n", + " \"No healthy workers found. Start them with:\\n\"\n", + " \" cd src/paddle_ocr && docker compose -f docker-compose.workers.yml up\"\n", + " )\n", + "\n", + "print(f\"\\n{len(healthy_workers)}/{len(WORKER_URLS)} workers ready for parallel tuning\")" + ] + }, + { + "cell_type": "markdown", + "id": "search-space-header", + "metadata": {}, + "source": [ + "## 3. Search Space" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "search-space", + "metadata": {}, + "outputs": [], + "source": [ + "search_space = {\n", + " # Whether to use document image orientation classification\n", + " \"use_doc_orientation_classify\": tune.choice([True, False]),\n", + " # Whether to use text image unwarping\n", + " \"use_doc_unwarping\": tune.choice([True, False]),\n", + " # Whether to use text line orientation classification\n", + " \"textline_orientation\": tune.choice([True, False]),\n", + " # Detection pixel threshold (pixels > threshold are considered text)\n", + " \"text_det_thresh\": tune.uniform(0.0, 0.7),\n", + " # Detection box threshold (average score within border)\n", + " \"text_det_box_thresh\": tune.uniform(0.0, 0.7),\n", + " # Text detection expansion coefficient\n", + " \"text_det_unclip_ratio\": tune.choice([0.0]),\n", + " # Text recognition threshold (filter low confidence results)\n", + " \"text_rec_score_thresh\": tune.uniform(0.0, 0.7),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "trainable-header", + "metadata": {}, + "source": [ + "## 4. Trainable Function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "trainable", + "metadata": {}, + "outputs": [], + "source": [ + "def trainable_paddle_ocr(config):\n", + " \"\"\"Call PaddleOCR REST API with the given hyperparameter config.\n", + " \n", + " Uses trial index to deterministically assign a worker (round-robin),\n", + " ensuring only 1 request per container at a time.\n", + " \"\"\"\n", + " import requests # Must be inside function for Ray workers\n", + " from ray import train\n", + "\n", + " # Worker URLs - round-robin assignment based on trial index\n", + " WORKER_PORTS = [8001, 8002]\n", + " NUM_WORKERS = len(WORKER_PORTS)\n", + " \n", + " # Get trial context for deterministic worker assignment\n", + " context = train.get_context()\n", + " trial_id = context.get_trial_id() if context else \"0\"\n", + " # Extract numeric part from trial ID (e.g., \"trainable_paddle_ocr_abc123_00001\" -> 1)\n", + " try:\n", + " trial_num = int(trial_id.split(\"_\")[-1])\n", + " except (ValueError, IndexError):\n", + " trial_num = hash(trial_id)\n", + " \n", + " worker_idx = trial_num % NUM_WORKERS\n", + " api_url = f\"http://localhost:{WORKER_PORTS[worker_idx]}\"\n", + "\n", + " payload = {\n", + " \"pdf_folder\": \"/app/dataset\",\n", + " \"use_doc_orientation_classify\": config.get(\"use_doc_orientation_classify\", False),\n", + " \"use_doc_unwarping\": config.get(\"use_doc_unwarping\", False),\n", + " \"textline_orientation\": config.get(\"textline_orientation\", True),\n", + " \"text_det_thresh\": config.get(\"text_det_thresh\", 0.0),\n", + " \"text_det_box_thresh\": config.get(\"text_det_box_thresh\", 0.0),\n", + " \"text_det_unclip_ratio\": config.get(\"text_det_unclip_ratio\", 1.5),\n", + " \"text_rec_score_thresh\": config.get(\"text_rec_score_thresh\", 0.0),\n", + " \"start_page\": 5,\n", + " \"end_page\": 10,\n", + " }\n", + "\n", + " try:\n", + " response = requests.post(f\"{api_url}/evaluate\", json=payload, timeout=None) # No timeout\n", + " response.raise_for_status()\n", + " metrics = response.json()\n", + " metrics[\"worker\"] = api_url\n", + " train.report(metrics)\n", + " except Exception as e:\n", + " train.report({\n", + " \"CER\": 1.0,\n", + " \"WER\": 1.0,\n", + " \"TIME\": 0.0,\n", + " \"PAGES\": 0,\n", + " \"TIME_PER_PAGE\": 0,\n", + " \"worker\": api_url,\n", + " \"ERROR\": str(e)[:500]\n", + " })" + ] + }, + { + "cell_type": "markdown", + "id": "tuner-header", + "metadata": {}, + "source": [ + "## 5. Run Tuner" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ray-init", + "metadata": {}, + "outputs": [], + "source": [ + "ray.init(ignore_reinit_error=True)\n", + "print(f\"Ray Tune ready (version: {ray.__version__})\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "tuner", + "metadata": {}, + "outputs": [], + "source": [ + "tuner = tune.Tuner(\n", + " trainable_paddle_ocr,\n", + " tune_config=tune.TuneConfig(\n", + " metric=\"CER\",\n", + " mode=\"min\",\n", + " search_alg=OptunaSearch(),\n", + " num_samples=64,\n", + " max_concurrent_trials=NUM_WORKERS, # Run trials in parallel across workers\n", + " ),\n", + " run_config=air.RunConfig(verbose=2, log_to_file=False),\n", + " param_space=search_space,\n", + ")\n", + "\n", + "results = tuner.fit()" + ] + }, + { + "cell_type": "markdown", + "id": "analysis-header", + "metadata": {}, + "source": [ + "## 6. Results Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "results-df", + "metadata": {}, + "outputs": [], + "source": [ + "df = results.get_dataframe()\n", + "df.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "save-results", + "metadata": {}, + "outputs": [], + "source": [ + "# Save results to CSV\n", + "timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n", + "filename = f\"raytune_paddle_rest_results_{timestamp}.csv\"\n", + "filepath = os.path.join(OUTPUT_FOLDER, filename)\n", + "\n", + "df.to_csv(filepath, index=False)\n", + "print(f\"Results saved: {filepath}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "best-config", + "metadata": {}, + "outputs": [], + "source": [ + "# Best configuration\n", + "best = df.loc[df[\"CER\"].idxmin()]\n", + "\n", + "print(f\"Best CER: {best['CER']:.6f}\")\n", + "print(f\"Best WER: {best['WER']:.6f}\")\n", + "print(f\"\\nOptimal Configuration:\")\n", + "print(f\" textline_orientation: {best['config/textline_orientation']}\")\n", + "print(f\" use_doc_orientation_classify: {best['config/use_doc_orientation_classify']}\")\n", + "print(f\" use_doc_unwarping: {best['config/use_doc_unwarping']}\")\n", + "print(f\" text_det_thresh: {best['config/text_det_thresh']:.4f}\")\n", + "print(f\" text_det_box_thresh: {best['config/text_det_box_thresh']:.4f}\")\n", + "print(f\" text_det_unclip_ratio: {best['config/text_det_unclip_ratio']}\")\n", + "print(f\" text_rec_score_thresh: {best['config/text_rec_score_thresh']:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "correlation", + "metadata": {}, + "outputs": [], + "source": [ + "# Correlation analysis\n", + "param_cols = [\n", + " \"config/text_det_thresh\",\n", + " \"config/text_det_box_thresh\",\n", + " \"config/text_det_unclip_ratio\",\n", + " \"config/text_rec_score_thresh\",\n", + "]\n", + "\n", + "corr_cer = df[param_cols + [\"CER\"]].corr()[\"CER\"].sort_values(ascending=False)\n", + "corr_wer = df[param_cols + [\"WER\"]].corr()[\"WER\"].sort_values(ascending=False)\n", + "\n", + "print(\"Correlation with CER:\")\n", + "print(corr_cer)\n", + "print(\"\\nCorrelation with WER:\")\n", + "print(corr_wer)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}