feat: add lighteval support

.github/workflows/build-and-push-apptainer.yml

@@ -37,7 +37,7 @@ jobs:
 
       - name: Build SIF from definition file
         run: |
-          apptainer --verbose build --fakeroot eval_env-${{ matrix.image }}.sif apptainer/${{ matrix.image }}.def
+          apptainer --verbose build --mksquashfs-args="-comp gzip -Xcompression-level 1" --fakeroot eval_env-${{ matrix.image }}.sif apptainer/${{ matrix.image }}.def
 
       - name: Install Hugging Face Hub CLI
         run: pip install --upgrade "huggingface_hub"

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Install uv
-      uses: astral-sh/setup-uv@v3
+      uses: astral-sh/setup-uv@v7
       with:
         version: "latest"
 
@@ -40,7 +40,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Install uv
-      uses: astral-sh/setup-uv@v3
+      uses: astral-sh/setup-uv@v7
       with:
         version: "latest"
 

.gitignore

@@ -13,3 +13,4 @@
 **/*.egg-info
 **/*.csv
 **/uv.lock
+**/task_map_cache.json

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -18,10 +18,3 @@ repos:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
-
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
-    hooks:
-      - id: mypy
-        additional_dependencies: [types-PyYAML]
-        args: [--ignore-missing-imports]

AGENTS.md

@@ -0,0 +1,5 @@
+Rules:
+- no try...Except unless absolutely necessary
+- no unnecessary comments
+- don't worry about tests
+- if you need to run stuff, assume there is a .venv at the root of the project. you can also just use uv

README.md

@@ -7,7 +7,7 @@ A package for running OELLM CLI workflows across multiple HPC clusters using SLU
 - Restart failed evaluations (e.g., due to node failures) ✅ `oellm collect-results ... --reschedule true`
 - Interactive eval job/csv builder ✅ `oellm build-csv`
   - Recursively resolve local paths: pass a directory containing models and their nested intermediate checkpoints, will eval all checkpoints
-  - Support default task groups (cf `oellm/task-groups.yaml`)
+  - Support default task groups (cf `oellm/resources/task-groups.yaml`)
 
 ## Planned workflows
 - Sync and download evaluation results from all clusters via a shared data layer
@@ -21,7 +21,7 @@ A package for running OELLM CLI workflows across multiple HPC clusters using SLU
 
 ```bash
 # Install the package
-uv tool install --python 3.12 git+https://github.com/OpenEuroLLM/oellm-cli.git
+uv tool install git+https://github.com/OpenEuroLLM/oellm-cli.git
 
 # Run evaluations on multiple models and tasks
 oellm schedule-eval \
@@ -50,6 +50,10 @@ This will launch an interactive workflow where you can:
 - Configure n-shot settings
 - Preview and save your evaluation configuration
 
+The resulting CSV includes an additional `eval_suite` column that records which
+evaluation framework (e.g., `lm_eval` or `lighteval`) should be used for each
+task.
+
 Otherwise you can also directly schedule using a CSV file:
 ```bash
 oellm schedule-eval --eval_csv_path custom_evals.csv
@@ -104,7 +108,7 @@ The `oellm` package orchestrates distributed LLM evaluations through the followi
 
 ### 1. **Cluster Auto-Detection**
 - Automatically detects the current HPC cluster based on hostname patterns
-- Loads cluster-specific configurations from [`clusters.yaml`](oellm/clusters.yaml) including:
+- Loads cluster-specific configurations from [`clusters.yaml`](oellm/resources/clusters.yaml) including:
   - SLURM partition and account settings
   - Shared storage paths for models, datasets, and results
   - GPU allocation and queue limits

apptainer/jureca.def

@@ -2,24 +2,37 @@ Bootstrap: docker
 From: nvcr.io/nvidia/pytorch:25.06-py3
 
 %labels
-    Author      multi-cluster-eval
-    Description Apptainer image for JURECA cluster (converted from dockerfile)
+    Author      oellm-cli
+    Description Apptainer image for JURECA JSC cluster
 
 %post
-    # 1. Install uv package manager
-    curl -LsSf https://astral.sh/uv/install.sh | sh
-    echo 'export PATH=$HOME/.local/bin:$PATH' >> /etc/profile
+    # Install uv into a global bin
+    curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh
 
-    # Make uv visible for subsequent commands during build
-    export PATH=/root/.local/bin:$PATH
+    # Put uv-installed tool shims in a global bin too
+    export UV_TOOL_BIN_DIR=/usr/local/bin
+    uv --version
 
-    # 2. Install Python dependencies
     uv pip install --system --break-system-packages lm-eval \
         "transformers<=4.53.0" "datasets<4.0.0" wandb sentencepiece tiktoken accelerate
 
+    # Optional: keep tool envs under /opt to avoid $HOME
+    export UV_TOOL_DIR=/opt/uv-tools
+    uv tool install --python 3.12 "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@63424f4e795ecc577b90646381b374af3a627978"
+    uv pip install --system --break-system-packages nltk
+    mkdir -p /opt/nltk_data
+    python - <<'PY'
+import nltk
+nltk.download('punkt', download_dir='/opt/nltk_data')
+nltk.download('punkt_tab', download_dir='/opt/nltk_data')
+PY
+
 %environment
-    # Ensure uv is present inside the container runtime as well
-    export PATH=/root/.local/bin:$PATH
+    export PATH=/usr/local/bin:$PATH
+    export UV_TOOL_BIN_DIR=/usr/local/bin
+    export UV_TOOL_DIR=/opt/uv-tools
+    export NLTK_DATA=/opt/nltk_data
+
 
 %runscript
     exec bash "$@"

apptainer/leonardo.def

@@ -2,24 +2,36 @@ Bootstrap: docker
 From: nvcr.io/nvidia/pytorch:25.06-py3
 
 %labels
-    Author      multi-cluster-eval
-    Description Apptainer image for Leonardo cluster (converted from dockerfile)
+    Author      oellm-cli
+    Description Apptainer image for Leonardo cluster
 
 %post
-    # 1. Install uv package manager
-    curl -LsSf https://astral.sh/uv/install.sh | sh
-    echo 'export PATH=$HOME/.local/bin:$PATH' >> /etc/profile
+    # Install uv into a global bin
+    curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh
 
-    # Make uv visible for subsequent commands during build
-    export PATH=/root/.local/bin:$PATH
+    # Put uv-installed tool shims in a global bin too
+    export UV_TOOL_BIN_DIR=/usr/local/bin
+    uv --version
 
-    # 2. Install Python dependencies
     uv pip install --system --break-system-packages lm-eval \
         "transformers<=4.53.0" "datasets<4.0.0" wandb sentencepiece tiktoken accelerate
 
+    # Optional: keep tool envs under /opt to avoid $HOME
+    export UV_TOOL_DIR=/opt/uv-tools
+    uv tool install --python 3.12 "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@63424f4e795ecc577b90646381b374af3a627978"
+    uv pip install --system --break-system-packages nltk
+    mkdir -p /opt/nltk_data
+    python - <<'PY'
+import nltk
+nltk.download('punkt', download_dir='/opt/nltk_data')
+nltk.download('punkt_tab', download_dir='/opt/nltk_data')
+PY
+
 %environment
-    # Ensure uv is present inside the container runtime as well
-    export PATH=/root/.local/bin:$PATH
+    export PATH=/usr/local/bin:$PATH
+    export UV_TOOL_BIN_DIR=/usr/local/bin
+    export UV_TOOL_DIR=/opt/uv-tools
+    export NLTK_DATA=/opt/nltk_data
 
 %runscript
     exec bash "$@"

apptainer/lumi.def

@@ -2,24 +2,36 @@ Bootstrap: docker
 From: rocm/pytorch:rocm6.4.1_ubuntu24.04_py3.12_pytorch_release_2.7.1
 
 %labels
-    Author      multi-cluster-eval
-    Description Apptainer image for LUMI cluster (converted from dockerfile)
+    Author      oellm-cli
+    Description Apptainer image for LUMI cluster
 
 %post
-    # 1. Install uv package manager
-    curl -LsSf https://astral.sh/uv/install.sh | sh
-    echo 'export PATH=$HOME/.local/bin:$PATH' >> /etc/profile
+    # Install uv into a global bin
+    curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh
 
-    # Make uv visible for subsequent commands during build
-    export PATH=/root/.local/bin:$PATH
+    # Put uv-installed tool shims in a global bin too
+    export UV_TOOL_BIN_DIR=/usr/local/bin
+    uv --version
 
-    # 2. Install Python dependencies
     uv pip install --system --break-system-packages lm-eval \
         "transformers<=4.53.0" "datasets<4.0.0" wandb sentencepiece tiktoken accelerate
 
+    # Optional: keep tool envs under /opt to avoid $HOME
+    export UV_TOOL_DIR=/opt/uv-tools
+    uv tool install --python 3.12 "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@63424f4e795ecc577b90646381b374af3a627978"
+    uv pip install --system --break-system-packages nltk
+    mkdir -p /opt/nltk_data
+    python - <<'PY'
+import nltk
+nltk.download('punkt', download_dir='/opt/nltk_data')
+nltk.download('punkt_tab', download_dir='/opt/nltk_data')
+PY
+
 %environment
-    # Ensure uv is present inside the container runtime as well
-    export PATH=/root/.local/bin:$PATH
+    export PATH=/usr/local/bin:$PATH
+    export UV_TOOL_BIN_DIR=/usr/local/bin
+    export UV_TOOL_DIR=/opt/uv-tools
+    export NLTK_DATA=/opt/nltk_data
 
 %runscript
     exec bash "$@"