diff --git a/.github/workflows/train.yaml b/.github/workflows/train.yaml index a43e5a1..6ea0f8d 100644 --- a/.github/workflows/train.yaml +++ b/.github/workflows/train.yaml @@ -20,12 +20,20 @@ jobs: name: pp-2016-MU-Sim10b-${{ github.head_ref }} snakemake_dir: pidgan target: cache_container validate_all + main_runner: aiinfn-lamarrsim-4gpus model_prefix: pid additional_config: | storage_folder: pid-2016MagUp-Sim10b additional_profile: | - jobs: 1 + jobs: 4 + cores: 32 + + resources: + mem_mb: 500000 + millicpu: 32000 + gpu: 4 + connections_to_storage: 2 generated_model_path: /tmp/compiled_model.c secrets: diff --git a/pidgan/config/config.yaml b/pidgan/config/config.yaml index 69d762a..65a60ce 100644 --- a/pidgan/config/config.yaml +++ b/pidgan/config/config.yaml @@ -6,7 +6,7 @@ apptainer_cached_image: /tmp/lb-pidsim-train.sif sample: 2016MU storage_folder: tests temp_dir: /tmp -training_time_limit_seconds: 600 +training_time_limit_seconds: 1200 entries_for_validation: 1000000 nbconvert_args: diff --git a/pidgan/workflow/Snakefile b/pidgan/workflow/Snakefile index 470a692..dfa96b2 100644 --- a/pidgan/workflow/Snakefile +++ b/pidgan/workflow/Snakefile @@ -80,7 +80,8 @@ rule preprocessing: resources: cpu=8, mem_mb=32000, - gpu=0 + gpu=0, + connections_to_storage=1, shell: """ @@ -265,7 +266,8 @@ rule validate: resources: cpu=4, mem_mb=15000, - gpu=0 + gpu=0, + connections_to_storage=1, shell: """ diff --git a/pidgan/workflow/notebooks/Deploy.ipynb b/pidgan/workflow/notebooks/Deploy.ipynb index 5c87f26..e292fd8 100644 --- a/pidgan/workflow/notebooks/Deploy.ipynb +++ b/pidgan/workflow/notebooks/Deploy.ipynb @@ -4,7 +4,13 @@ "cell_type": "code", "execution_count": 1, "id": "73cdaffe-5f9a-421e-8108-009115be1b94", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "name": "stderr", @@ -23,7 +29,9 @@ "import scikinC\n", "from datetime import datetime\n", "import hashlib\n", - "from preprocessing_utils import ProbNNTransformer_fwd, ProbNNTransformer_bwd" + "from preprocessing_utils import ProbNNTransformer_fwd, ProbNNTransformer_bwd\n", + "from gpu_picker import request_gpu\n", + "print(\"Request gpu:\", request_gpu(False))" ] }, { diff --git a/pidgan/workflow/notebooks/GlobalPID-im-preprocessing.ipynb b/pidgan/workflow/notebooks/GlobalPID-im-preprocessing.ipynb index 837ac2d..c9b98db 100644 --- a/pidgan/workflow/notebooks/GlobalPID-im-preprocessing.ipynb +++ b/pidgan/workflow/notebooks/GlobalPID-im-preprocessing.ipynb @@ -3,7 +3,13 @@ { "cell_type": "markdown", "id": "28185ab6-475a-4e7e-82ed-9293c407146d", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Preprocessing steps for _GlobalPID_ models (for isMuon=True)" ] @@ -12,7 +18,13 @@ "cell_type": "code", "execution_count": 1, "id": "9330759e-6339-4cd9-bd04-8426edb4ebeb", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -33,6 +45,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "PARTICLE = os.environ.get(\"PARTICLE\", \"muon\")\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -42,6 +55,7 @@ " * PARTICLE: `{PARTICLE}`\n", " * LOCAL: `{LOCAL}`\n", " * SAMPLE: `{SAMPLE}`\n", + " * REQUESTED GPU: `{request_gpu(False)}`\n", "\"\"\")" ] }, diff --git a/pidgan/workflow/notebooks/GlobalPID-im-train.ipynb b/pidgan/workflow/notebooks/GlobalPID-im-train.ipynb index b515d6a..c37289c 100644 --- a/pidgan/workflow/notebooks/GlobalPID-im-train.ipynb +++ b/pidgan/workflow/notebooks/GlobalPID-im-train.ipynb @@ -3,7 +3,13 @@ { "cell_type": "markdown", "id": "496d2ab8-65b0-4113-bf74-edaed195fae2", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Training of the _GlobalPID_ model for particles passing the isMuon criterion\n", "\n", @@ -16,7 +22,13 @@ "cell_type": "code", "execution_count": 1, "id": "90f91f2f-e0d2-4873-8990-21116d927d03", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -39,6 +51,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", "TEST = os.environ.get(\"TEST\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -50,6 +63,7 @@ " * SAMPLE: `{SAMPLE}`\n", " * TEST: `{TEST}`\n", " * PARTICLE: `{PARTICLE}`\n", + " * REQUESTED GPU: `{request_gpu(True)}`\n", "\"\"\")" ] }, diff --git a/pidgan/workflow/notebooks/GlobalPID-nm-preprocessing.ipynb b/pidgan/workflow/notebooks/GlobalPID-nm-preprocessing.ipynb index d20b704..94fb606 100644 --- a/pidgan/workflow/notebooks/GlobalPID-nm-preprocessing.ipynb +++ b/pidgan/workflow/notebooks/GlobalPID-nm-preprocessing.ipynb @@ -3,7 +3,13 @@ { "cell_type": "markdown", "id": "9800a068-ce09-45a1-ae4a-6edec0747e13", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Preprocessing steps for _GlobalPID_ models (for isMuon=False)" ] @@ -12,7 +18,13 @@ "cell_type": "code", "execution_count": 1, "id": "f095e84c-a1ec-4ffc-9384-6cc06ef6eeb3", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -34,6 +46,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "PARTICLE = os.environ.get(\"PARTICLE\", \"muon\")\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -42,7 +55,8 @@ "Markdown(f\"\"\"\n", " * PARTICLE: `{PARTICLE}`\n", " * LOCAL: `{LOCAL}`\n", - " * SAMPLE: `{SAMPLE}`\n", + " * SAMPLE: `{SAMPLE}` \n", + " * REQUESTED GPU: `{request_gpu(False)}`\n", "\"\"\")" ] }, diff --git a/pidgan/workflow/notebooks/GlobalPID-nm-train.ipynb b/pidgan/workflow/notebooks/GlobalPID-nm-train.ipynb index 52d0283..9c9918e 100644 --- a/pidgan/workflow/notebooks/GlobalPID-nm-train.ipynb +++ b/pidgan/workflow/notebooks/GlobalPID-nm-train.ipynb @@ -3,7 +3,13 @@ { "cell_type": "markdown", "id": "496d2ab8-65b0-4113-bf74-edaed195fae2", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Training of the _GlobalPID_ model for particles **NOT** passing the isMuon criterion\n", "\n", @@ -16,7 +22,13 @@ "cell_type": "code", "execution_count": 1, "id": "90f91f2f-e0d2-4873-8990-21116d927d03", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -39,6 +51,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", "TEST = os.environ.get(\"TEST\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -49,6 +62,7 @@ " * LOCAL: `{LOCAL}`\n", " * SAMPLE: `{SAMPLE}`\n", " * TEST: `{TEST}`\n", + " * REQUESTED GPU: `{request_gpu(True)}`\n", " * PARTICLE: `{PARTICLE}`\n", "\"\"\")" ] diff --git a/pidgan/workflow/notebooks/Muon-preprocessing.ipynb b/pidgan/workflow/notebooks/Muon-preprocessing.ipynb index 84b6371..4f087ee 100644 --- a/pidgan/workflow/notebooks/Muon-preprocessing.ipynb +++ b/pidgan/workflow/notebooks/Muon-preprocessing.ipynb @@ -2,7 +2,13 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Preprocessing steps for _MUON_ models" ] @@ -10,12 +16,19 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { "text/markdown": [ "\n", + " * PARTICLE: `muon`\n", " * LOCAL: `True`\n", " * SAMPLE: `2016MU`\n" ], @@ -31,6 +44,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "PARTICLE = os.environ.get(\"PARTICLE\", \"muon\")\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -40,12 +54,19 @@ " * PARTICLE: `{PARTICLE}`\n", " * LOCAL: `{LOCAL}`\n", " * SAMPLE: `{SAMPLE}`\n", + " * REQUESTED GPU: `{request_gpu(False)}`\n", "\"\"\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "## Technologies and libraries\n", "\n", diff --git a/pidgan/workflow/notebooks/Muon-train.ipynb b/pidgan/workflow/notebooks/Muon-train.ipynb index c19f46c..d8ee763 100644 --- a/pidgan/workflow/notebooks/Muon-train.ipynb +++ b/pidgan/workflow/notebooks/Muon-train.ipynb @@ -3,7 +3,13 @@ { "cell_type": "markdown", "id": "496d2ab8-65b0-4113-bf74-edaed195fae2", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Training of the _Muon_ model \n", "\n", @@ -16,7 +22,13 @@ "cell_type": "code", "execution_count": 1, "id": "90f91f2f-e0d2-4873-8990-21116d927d03", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -39,6 +51,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", "TEST = os.environ.get(\"TEST\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -50,6 +63,8 @@ " * SAMPLE: `{SAMPLE}`\n", " * TEST: `{TEST}`\n", " * PARTICLE: `{PARTICLE}`\n", + " * REQUESTED GPU: `{request_gpu(True)}`\n", + "\n", "\"\"\")" ] }, diff --git a/pidgan/workflow/notebooks/Rich-preprocessing.ipynb b/pidgan/workflow/notebooks/Rich-preprocessing.ipynb index ff50513..468ed35 100644 --- a/pidgan/workflow/notebooks/Rich-preprocessing.ipynb +++ b/pidgan/workflow/notebooks/Rich-preprocessing.ipynb @@ -2,15 +2,27 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Preprocessing steps for _RICH_ models" ] }, { "cell_type": "code", - "execution_count": 1, - "metadata": {}, + "execution_count": 2, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -18,13 +30,14 @@ "\n", " * PARTICLE: `muon`\n", " * LOCAL: `True`\n", - " * SAMPLE: `2016MU`\n" + " * SAMPLE: `2016MU`\n", + " * REQUESTED GPU: `None`\n" ], "text/plain": [ "" ] }, - "execution_count": 1, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -32,6 +45,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "PARTICLE = os.environ.get(\"PARTICLE\", \"muon\")\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -41,12 +55,19 @@ " * PARTICLE: `{PARTICLE}`\n", " * LOCAL: `{LOCAL}`\n", " * SAMPLE: `{SAMPLE}`\n", + " * REQUESTED GPU: `{request_gpu(False)}`\n", "\"\"\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "## Technologies and libraries\n", "\n", diff --git a/pidgan/workflow/notebooks/Rich-train.ipynb b/pidgan/workflow/notebooks/Rich-train.ipynb index b3e2245..668b027 100644 --- a/pidgan/workflow/notebooks/Rich-train.ipynb +++ b/pidgan/workflow/notebooks/Rich-train.ipynb @@ -3,7 +3,13 @@ { "cell_type": "markdown", "id": "496d2ab8-65b0-4113-bf74-edaed195fae2", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Training of the _RICH_ model\n", "\n", @@ -16,7 +22,13 @@ "cell_type": "code", "execution_count": 1, "id": "90f91f2f-e0d2-4873-8990-21116d927d03", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -39,6 +51,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", "TEST = os.environ.get(\"TEST\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -50,6 +63,7 @@ " * SAMPLE: `{SAMPLE}`\n", " * TEST: `{TEST}`\n", " * PARTICLE: `{PARTICLE}`\n", + " * REQUESTED GPU: `{request_gpu(True)}`\n", "\"\"\")" ] }, diff --git a/pidgan/workflow/notebooks/isMuon-preprocessing.ipynb b/pidgan/workflow/notebooks/isMuon-preprocessing.ipynb index f543cda..072f3ee 100644 --- a/pidgan/workflow/notebooks/isMuon-preprocessing.ipynb +++ b/pidgan/workflow/notebooks/isMuon-preprocessing.ipynb @@ -2,7 +2,13 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Preprocessing steps for _isMuon_ models" ] @@ -10,7 +16,13 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -31,6 +43,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "PARTICLE = os.environ.get(\"PARTICLE\", \"muon\")\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -40,6 +53,7 @@ " * PARTICLE: `{PARTICLE}`\n", " * LOCAL: `{LOCAL}`\n", " * SAMPLE: `{SAMPLE}`\n", + " * REQUESTED GPU: `{request_gpu(False)}`\n", "\"\"\")" ] }, diff --git a/pidgan/workflow/notebooks/isMuon-train.ipynb b/pidgan/workflow/notebooks/isMuon-train.ipynb index f9a976b..d0b5255 100644 --- a/pidgan/workflow/notebooks/isMuon-train.ipynb +++ b/pidgan/workflow/notebooks/isMuon-train.ipynb @@ -3,7 +3,13 @@ { "cell_type": "markdown", "id": "496d2ab8-65b0-4113-bf74-edaed195fae2", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Training of the _isMuon_ model \n", "\n", @@ -16,7 +22,13 @@ "cell_type": "code", "execution_count": 1, "id": "90f91f2f-e0d2-4873-8990-21116d927d03", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -39,6 +51,7 @@ "source": [ "import os\n", "from IPython.display import Markdown\n", + "from gpu_picker import request_gpu\n", "\n", "LOCAL = os.environ.get(\"LOCAL\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", "TEST = os.environ.get(\"TEST\", \"yes\").lower() in ['y', 'yes', 'true', '1']\n", @@ -50,6 +63,7 @@ " * SAMPLE: `{SAMPLE}`\n", " * TEST: `{TEST}`\n", " * PARTICLE: `{PARTICLE}`\n", + " * REQUESTED GPU: `{request_gpu(True)}`\n", "\"\"\")" ] }, diff --git a/pidgan/workflow/notebooks/validate.ipynb b/pidgan/workflow/notebooks/validate.ipynb index 740f90d..8cbf0db 100644 --- a/pidgan/workflow/notebooks/validate.ipynb +++ b/pidgan/workflow/notebooks/validate.ipynb @@ -3,16 +3,28 @@ { "cell_type": "markdown", "id": "328dd053-033d-402d-9292-5daf69ad230a", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# Validation" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "3a7d3458-49f0-4863-a031-a214dfc1f32e", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -22,13 +34,15 @@ " * LOCAL: `True`\n", " * SAMPLE: `2016MU`\n", " * SHARED_OBJECT: `/tmp/exported/generated1756102685.532006.so`\n", - " * MAX_ENTRIES: `10000`\n" + " * MAX_ENTRIES: `10000`\n", + " * REQUESTED GPU: `None`\n", + "\n" ], "text/plain": [ "" ] }, - "execution_count": 3, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -41,6 +55,8 @@ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import mplhep\n", + "from gpu_picker import request_gpu\n", + "\n", "\n", "mplhep.style.use(\"LHCb2\")\n", "\n", @@ -59,6 +75,8 @@ " * SAMPLE: `{SAMPLE}`\n", " * SHARED_OBJECT: `{SHARED_OBJECT}`\n", " * MAX_ENTRIES: `{MAX_ENTRIES}`\n", + " * REQUESTED GPU: `{request_gpu(False)}`\n", + " \n", "\"\"\")\n" ] }, @@ -66,7 +84,13 @@ "cell_type": "code", "execution_count": 2, "id": "189846d6-d3cd-4af3-9395-da083c404a09", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [], "source": [ "from scikinC.validation import MLFunction\n",