From b35cee4a2c0076d63fab525e2d8e929adda58cc1 Mon Sep 17 00:00:00 2001 From: Maurice Kraus Date: Mon, 23 Feb 2026 15:29:07 +0000 Subject: [PATCH 1/2] fix: correct GPU GRES parsing and add regression tests --- .github/workflows/ci.yml | 26 +++++++++ fixtures/scontrol.json | 3 ++ pyproject.toml | 1 + src/slurm_backend.py | 68 +++++++++++++++--------- tests/conftest.py | 6 +++ tests/fixture_loader.py | 11 ++++ tests/test_slurm_backend_job_parsing.py | 31 +++++++++++ tests/test_slurm_backend_node_parsing.py | 39 ++++++++++++++ 8 files changed, 160 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 fixtures/scontrol.json create mode 100644 tests/conftest.py create mode 100644 tests/fixture_loader.py create mode 100644 tests/test_slurm_backend_job_parsing.py create mode 100644 tests/test_slurm_backend_node_parsing.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d0642e2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + run: uv python install 3.10 + + - name: Install dependencies + run: uv sync --dev + + - name: Run tests + run: PYTHONPATH=src uv run pytest -q diff --git a/fixtures/scontrol.json b/fixtures/scontrol.json new file mode 100644 index 0000000..ce93e4c --- /dev/null +++ b/fixtures/scontrol.json @@ -0,0 +1,3 @@ +{ + "show_node": "NodeName=cn01 Arch=x86_64 CoresPerSocket=56 CPUAlloc=48 CPUEfctv=112 CPUTot=112 CPULoad=5.05 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn01 NodeHostName=cn01 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=217088 FreeMem=166705 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T15:35:53 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2025-11-18T11:28:44 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=48,mem=212G,gres/gpu:h100=6 CurrentWatts=4500 AveWatts=1458 \nNodeName=cn02 Arch=x86_64 CoresPerSocket=56 CPUAlloc=47 CPUEfctv=112 CPUTot=112 CPULoad=4.00 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn02 NodeHostName=cn02 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=294912 FreeMem=134457 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T09:36:34 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-19T09:47:08 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=47,mem=288G,gres/gpu:h100=4 CurrentWatts=4275 AveWatts=3674 \nNodeName=cn03 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=0.01 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn03 NodeHostName=cn03 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=224506 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:56 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-10T03:03:58 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=1T,gres/gpu:h100=8 CurrentWatts=1725 AveWatts=1500 \nNodeName=cn04 Arch=x86_64 CoresPerSocket=56 CPUAlloc=49 CPUEfctv=112 CPUTot=112 CPULoad=3.09 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn04 NodeHostName=cn04 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=737280 FreeMem=843119 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:54 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-18T21:00:51 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=49,mem=720G,gres/gpu:h100=7 CurrentWatts=2025 AveWatts=1869 \nNodeName=cn05 Arch=x86_64 CoresPerSocket=56 CPUAlloc=1 CPUEfctv=112 CPUTot=112 CPULoad=13.25 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn05 NodeHostName=cn05 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=1648339 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2026-02-16T17:21:08 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-16T17:39:40 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=1,mem=1T,gres/gpu:h100=8 CurrentWatts=4950 AveWatts=1869 \nNodeName=cn06 Arch=x86_64 CoresPerSocket=56 CPUAlloc=32 CPUEfctv=112 CPUTot=112 CPULoad=3.12 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn06 NodeHostName=cn06 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=229376 FreeMem=78827 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:56 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-01-22T14:53:47 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=32,mem=224G,gres/gpu:h100=3 CurrentWatts=2250 AveWatts=1650 \nNodeName=cn07 Arch=x86_64 CoresPerSocket=56 CPUAlloc=8 CPUEfctv=112 CPUTot=112 CPULoad=13.40 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn07 NodeHostName=cn07 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=49246 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:21:01 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-06T03:36:45 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=8,mem=1T,gres/gpu:h100=8 CurrentWatts=3300 AveWatts=2424 \nNodeName=cn08 Arch=x86_64 CoresPerSocket=56 CPUAlloc=86 CPUEfctv=112 CPUTot=112 CPULoad=1.07 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn08 NodeHostName=cn08 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=458752 FreeMem=697315 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:55 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-22T09:04:32 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=86,mem=448G,gres/gpu:h100=6 CurrentWatts=1875 AveWatts=1628 \nNodeName=cn09 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=19.99 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn09 NodeHostName=cn09 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=2063400 FreeMem=6244 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:55 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-12T22:51:30 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=2063400M,gres/gpu:h100=8 CurrentWatts=3450 AveWatts=2615 \nNodeName=cn10 Arch=x86_64 CoresPerSocket=56 CPUAlloc=1 CPUEfctv=112 CPUTot=112 CPULoad=8.03 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn10 NodeHostName=cn10 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=262144 FreeMem=25486 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:54 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-19T19:27:38 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=1,mem=256G,gres/gpu:h100=8 CurrentWatts=2100 AveWatts=1950 \nNodeName=cn11 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=8.02 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn11 NodeHostName=cn11 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=2063400 FreeMem=153386 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:57 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T04:39:34 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=2063400M,gres/gpu:h100=8 CurrentWatts=2475 AveWatts=1566 \nNodeName=cn12 Arch=x86_64 CoresPerSocket=56 CPUAlloc=1 CPUEfctv=112 CPUTot=112 CPULoad=8.03 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn12 NodeHostName=cn12 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=131072 FreeMem=1132501 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:21:04 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-20T17:21:14 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=1,mem=128G,gres/gpu:h100=8 CurrentWatts=2175 AveWatts=2002 \nNodeName=cn13 Arch=x86_64 CoresPerSocket=56 CPUAlloc=0 CPUEfctv=112 CPUTot=112 CPULoad=0.00 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn13 NodeHostName=cn13 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=0 FreeMem=75939 Sockets=2 Boards=1 State=IDLE ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=short BootTime=2025-12-01T12:08:18 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-20T17:43:18 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES= CurrentWatts=2250 AveWatts=2100 \nNodeName=cn14 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=9.44 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn14 NodeHostName=cn14 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=15902 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:54 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T09:37:12 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=1T,gres/gpu:h100=8 CurrentWatts=2250 AveWatts=1976 \nNodeName=cn15 Arch=x86_64 CoresPerSocket=56 CPUAlloc=32 CPUEfctv=112 CPUTot=112 CPULoad=8.23 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn15 NodeHostName=cn15 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1024000 FreeMem=7735 Sockets=2 Boards=1 State=MIXED+PLANNED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:28:30 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-21T20:14:10 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=32,mem=1000G,gres/gpu:h100=8 CurrentWatts=2250 AveWatts=2167 \nNodeName=cn16 Arch=x86_64 CoresPerSocket=56 CPUAlloc=56 CPUEfctv=112 CPUTot=112 CPULoad=4.02 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn16 NodeHostName=cn16 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=122880 FreeMem=114280 Sockets=2 Boards=1 State=MIXED+PLANNED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:53 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-20T13:53:40 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=56,mem=120G,gres/gpu:h100=4 CurrentWatts=1950 AveWatts=2062 \nNodeName=cn17 Arch=x86_64 CoresPerSocket=56 CPUAlloc=40 CPUEfctv=112 CPUTot=112 CPULoad=8.34 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn17 NodeHostName=cn17 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1056768 FreeMem=38639 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:21:15 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-21T20:14:10 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=40,mem=1032G,gres/gpu:h100=8 CurrentWatts=2325 AveWatts=2013 \nNodeName=cn18 Arch=x86_64 CoresPerSocket=56 CPUAlloc=0 CPUEfctv=112 CPUTot=112 CPULoad=0.00 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn18 NodeHostName=cn18 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=0 FreeMem=320950 Sockets=2 Boards=1 State=IDLE ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=short BootTime=2025-11-17T10:20:55 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-20T17:43:18 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES= CurrentWatts=1575 AveWatts=1568 \nNodeName=cn19 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=11.31 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn19 NodeHostName=cn19 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=203670 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:55 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T10:12:50 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=1T,gres/gpu:h100=8 CurrentWatts=6375 AveWatts=1570 \nNodeName=cn20 Arch=x86_64 CoresPerSocket=56 CPUAlloc=4 CPUEfctv=112 CPUTot=112 CPULoad=1.32 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn20 NodeHostName=cn20 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=65536 FreeMem=415412 Sockets=2 Boards=1 State=MIXED+PLANNED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:55 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T00:12:38 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=4,mem=64G,gres/gpu:h100=1 CurrentWatts=5400 AveWatts=3037 \nNodeName=cn21 Arch=x86_64 CoresPerSocket=56 CPUAlloc=1 CPUEfctv=112 CPUTot=112 CPULoad=1.79 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn21 NodeHostName=cn21 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=2063400 FreeMem=25738 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:59 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T14:40:10 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=1,mem=2063400M,gres/gpu:h100=1 CurrentWatts=1575 AveWatts=2776 \nNodeName=cn22 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=11.90 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn22 NodeHostName=cn22 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=317164 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:54 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T00:12:34 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=1T,gres/gpu:h100=8 CurrentWatts=6300 AveWatts=3626 \nNodeName=cn23 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=11.00 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn23 NodeHostName=cn23 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=30616 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:58 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T00:12:34 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=1T,gres/gpu:h100=8 CurrentWatts=7275 AveWatts=2301 \nNodeName=cn24 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=12.01 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn24 NodeHostName=cn24 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=44078 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:53 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T00:12:34 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=1T,gres/gpu:h100=8 CurrentWatts=6375 AveWatts=3320 \nNodeName=cn25 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=12.44 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn25 NodeHostName=cn25 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=59324 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:56 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T00:12:38 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=1T,gres/gpu:h100=8 CurrentWatts=6375 AveWatts=3645 \nNodeName=cn26 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=11.96 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn26 NodeHostName=cn26 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=64863 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:56 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T00:12:34 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=1T,gres/gpu:h100=8 CurrentWatts=6525 AveWatts=3624 \nNodeName=cn27 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=11.06 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn27 NodeHostName=cn27 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=352941 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:55 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T00:12:34 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=1T,gres/gpu:h100=8 CurrentWatts=6450 AveWatts=1500 \nNodeName=cn28 Arch=x86_64 CoresPerSocket=56 CPUAlloc=32 CPUEfctv=112 CPUTot=112 CPULoad=8.12 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn28 NodeHostName=cn28 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1024000 FreeMem=17705 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:55 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-22T20:35:52 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=32,mem=1000G,gres/gpu:h100=8 CurrentWatts=2400 AveWatts=2207 \nNodeName=cn29 Arch=x86_64 CoresPerSocket=56 CPUAlloc=32 CPUEfctv=112 CPUTot=112 CPULoad=8.10 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn29 NodeHostName=cn29 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1024000 FreeMem=29344 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:56 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-22T20:35:52 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=32,mem=1000G,gres/gpu:h100=8 CurrentWatts=2175 AveWatts=2092 \nNodeName=cn30 Arch=x86_64 CoresPerSocket=56 CPUAlloc=84 CPUEfctv=112 CPUTot=112 CPULoad=6.00 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn30 NodeHostName=cn30 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=768000 FreeMem=100055 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:57 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-23T03:16:19 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=84,mem=750G,gres/gpu:h100=6 CurrentWatts=4350 AveWatts=2066 \nNodeName=cn31 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=8.02 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn31 NodeHostName=cn31 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=245760 FreeMem=700514 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:54 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-20T18:34:57 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=240G,gres/gpu:h100=8 CurrentWatts=2175 AveWatts=2175 \nNodeName=cn32 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=21.33 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn32 NodeHostName=cn32 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=2063400 FreeMem=5754 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:54 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-05T17:46:50 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=2063400M,gres/gpu:h100=8 CurrentWatts=2925 AveWatts=2605 \nNodeName=cn33 Arch=x86_64 CoresPerSocket=56 CPUAlloc=32 CPUEfctv=112 CPUTot=112 CPULoad=4.06 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn33 NodeHostName=cn33 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=262144 FreeMem=75618 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:54 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-19T19:42:56 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=32,mem=256G,gres/gpu:h100=8 CurrentWatts=2175 AveWatts=2100 \nNodeName=cn34 Arch=x86_64 CoresPerSocket=56 CPUAlloc=64 CPUEfctv=112 CPUTot=112 CPULoad=59.90 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn34 NodeHostName=cn34 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=1048576 FreeMem=34429 Sockets=2 Boards=1 State=MIXED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:54 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-22T17:24:47 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=64,mem=1T,gres/gpu:h100=8 CurrentWatts=5250 AveWatts=2045 \nNodeName=cn35 Arch=x86_64 CoresPerSocket=56 CPUAlloc=112 CPUEfctv=112 CPUTot=112 CPULoad=70.19 AvailableFeatures=(null) ActiveFeatures=(null) Gres=gpu:h100:8(S:0-1) NodeAddr=cn35 NodeHostName=cn35 Version=24.11.4 OS=Linux 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 1 19:11:28 UTC 2024 RealMemory=2063400 AllocMem=2063400 FreeMem=8269 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=0 Weight=1 Owner=N/A MCS_label=N/A Partitions=all BootTime=2025-11-17T10:20:57 SlurmdStartTime=2026-02-20T15:07:14 LastBusyTime=2026-02-11T00:51:45 ResumeAfterTime=None CfgTRES=cpu=112,mem=2063400M,billing=112,gres/gpu:h100=8 AllocTRES=cpu=112,mem=2063400M,gres/gpu:h100=8 CurrentWatts=2700 AveWatts=2160" +} diff --git a/pyproject.toml b/pyproject.toml index 21e4513..6709602 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ [dependency-groups] dev = [ "pyinstaller>=6.18.0", + "pytest>=8.4.0", ] [project.scripts] diff --git a/src/slurm_backend.py b/src/slurm_backend.py index d514cb1..585161d 100644 --- a/src/slurm_backend.py +++ b/src/slurm_backend.py @@ -4,6 +4,36 @@ from fake_slurm_fixtures import run_fake_slurm_command from smon_config import USE_FAKE_DATA + +GPU_GRES_COUNT_PATTERN = re.compile( + r"(?:^|,)\s*(?:gres/)?gpu:(?:(?:[^:,()\s]+):)?(\d+)(?=$|,|\()" +) +GPU_ALLOC_TRES_PATTERN = re.compile(r"(?:^|,)\s*gres/gpu(?::[^=,\s]+)?=(\d+)(?=$|,)") +GPU_TOTAL_PATTERN = re.compile(r"(?:gpu_total|total_gpu)[:=](\d+)") + + +def _parse_gres_gpu_count(gres_value: str) -> int: + return sum(int(count) for count in GPU_GRES_COUNT_PATTERN.findall(gres_value)) + + +def _parse_alloc_tres_gpu_count(alloc_tres: str) -> int: + return sum(int(count) for count in GPU_ALLOC_TRES_PATTERN.findall(alloc_tres)) + + +def _parse_gpu_total(gpu_field: str) -> int | None: + match = GPU_TOTAL_PATTERN.search(gpu_field) + if match: + return int(match.group(1)) + return None + + +def _parse_gpu_per_node(gpu_field: str) -> int | None: + counts = GPU_GRES_COUNT_PATTERN.findall(gpu_field) + if counts: + return sum(int(count) for count in counts) + return None + + def run_slurm_command(cmd: str) -> str: if USE_FAKE_DATA: return run_fake_slurm_command(cmd) @@ -35,18 +65,10 @@ def get_cluster_stats(): m_u = int(data.get("AllocMem", 0)) m_t = int(data.get("RealMemory", 1)) - g_t, g_u = 0, 0 gres_str = data.get("Gres", "") - if "gpu" in gres_str: - parts = re.findall(r":(\d+)", gres_str) - if parts: - g_t = int(parts[0]) - alloc_tres = data.get("AllocTRES", "") - if "gres/gpu" in alloc_tres: - match = re.search(r"gres/gpu[^=]*=(\d+)", alloc_tres) - if match: - g_u = int(match.group(1)) + g_t = _parse_gres_gpu_count(gres_str) + g_u = _parse_alloc_tres_gpu_count(alloc_tres) t_cpu_u += c_u t_cpu_t += c_t @@ -97,22 +119,18 @@ def get_job_stats(): gpu_count = "-" gpu_field = parts[8] - if "gpu" in gpu_field: - try: - # Support explicit total notation in fixtures: gpu_total=64 - total_match = re.search( - r"(?:gpu_total|total_gpu|gres/gpu)[:=](\d+)", gpu_field - ) - if total_match: - gpu_count = total_match.group(1) - else: + try: + # Fixture compatibility: explicit total markers are whole-job totals. + explicit_total = _parse_gpu_total(gpu_field) + if explicit_total is not None: + gpu_count = str(explicit_total) + else: + per_node = _parse_gpu_per_node(gpu_field) + if per_node is not None: node_mult = int(parts[6]) - per_node_match = re.search(r"gpu[^0-9]*(\d+)", gpu_field) - if per_node_match: - per_node = int(per_node_match.group(1)) - gpu_count = str(node_mult * per_node) - except Exception: - pass + gpu_count = str(node_mult * per_node) + except Exception: + pass dep = parts[16] if dep == "(null)" or dep == "N/A": diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8f53317 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,6 @@ +from pathlib import Path +import sys + +SRC_PATH = Path(__file__).resolve().parents[1] / "src" +if str(SRC_PATH) not in sys.path: + sys.path.insert(0, str(SRC_PATH)) diff --git a/tests/fixture_loader.py b/tests/fixture_loader.py new file mode 100644 index 0000000..6d17f2b --- /dev/null +++ b/tests/fixture_loader.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import json +from pathlib import Path + +FIXTURE_PATH = Path(__file__).resolve().parents[1] / "fixtures" / "scontrol.json" + + +def load_scontrol_show_node_output() -> str: + data = json.loads(FIXTURE_PATH.read_text()) + return data["show_node"] diff --git a/tests/test_slurm_backend_job_parsing.py b/tests/test_slurm_backend_job_parsing.py new file mode 100644 index 0000000..0589d52 --- /dev/null +++ b/tests/test_slurm_backend_job_parsing.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import slurm_backend + + +def _build_squeue_output() -> str: + header = ( + "JOBID USER STATE TIME LEFT PRIO NODES REASON GRES NAME CPU MEM PART " + "ACCOUNT QOS SUBMIT DEP" + ) + rows = [ + "100001 user01 RUNNING 00:10:00 00:50:00 100 1 cn01 gres/gpu:h100:4 train_a 64 256G all acct01 normal 2026-02-23T10:00:00 (null)", + "100002 user02 RUNNING 00:15:00 00:45:00 110 2 cn02 gres/gpu:h100:1 train_b 64 256G all acct02 normal 2026-02-23T10:05:00 (null)", + "100003 user03 RUNNING 00:05:00 00:55:00 120 1 cn03 gres/gpu:1 train_c 64 256G all acct03 normal 2026-02-23T10:10:00 (null)", + "100004 user04 RUNNING 00:20:00 00:40:00 130 4 cn04 gpu_total=64 train_d 64 256G all acct04 normal 2026-02-23T10:15:00 (null)", + "100005 user05 RUNNING 00:25:00 00:35:00 140 1 cn05 (null) train_e 64 256G all acct05 normal 2026-02-23T10:20:00 (null)", + ] + return "\n".join([header, *rows]) + + +def test_get_job_stats_parses_typed_and_untyped_gpu_counts(monkeypatch): + monkeypatch.setattr(slurm_backend, "run_slurm_command", lambda _cmd: _build_squeue_output()) + + jobs = slurm_backend.get_job_stats() + jobs_by_id = {job["id"]: job for job in jobs} + + assert jobs_by_id["100001"]["gpu"] == "4" + assert jobs_by_id["100002"]["gpu"] == "2" + assert jobs_by_id["100003"]["gpu"] == "1" + assert jobs_by_id["100004"]["gpu"] == "64" + assert jobs_by_id["100005"]["gpu"] == "-" diff --git a/tests/test_slurm_backend_node_parsing.py b/tests/test_slurm_backend_node_parsing.py new file mode 100644 index 0000000..fab47c4 --- /dev/null +++ b/tests/test_slurm_backend_node_parsing.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import slurm_backend + +from fixture_loader import load_scontrol_show_node_output + + +def test_get_cluster_stats_parses_real_scontrol_fixture(monkeypatch): + show_node_output = load_scontrol_show_node_output() + monkeypatch.setattr(slurm_backend, "run_slurm_command", lambda _cmd: show_node_output) + + nodes, theoretical, real = slurm_backend.get_cluster_stats() + + assert len(nodes) == 35 + assert theoretical == (2218, 3920, 230, 280) + assert real == (3920, 280) + + +def test_get_cluster_stats_parses_representative_gpu_nodes(monkeypatch): + show_node_output = load_scontrol_show_node_output() + monkeypatch.setattr(slurm_backend, "run_slurm_command", lambda _cmd: show_node_output) + + nodes, _theoretical, _real = slurm_backend.get_cluster_stats() + by_name = {node["name"]: node for node in nodes} + + assert by_name["cn01"]["g_t"] == 8 + assert by_name["cn01"]["g_u"] == 6 + + assert by_name["cn02"]["g_t"] == 8 + assert by_name["cn02"]["g_u"] == 4 + + assert by_name["cn06"]["g_t"] == 8 + assert by_name["cn06"]["g_u"] == 3 + + assert by_name["cn20"]["g_t"] == 8 + assert by_name["cn20"]["g_u"] == 1 + + assert by_name["cn13"]["g_u"] == 0 + assert by_name["cn18"]["g_u"] == 0 From d7744cf69889fe2b7948fe58fe907c7ed74d7f02 Mon Sep 17 00:00:00 2001 From: Maurice Kraus Date: Mon, 23 Feb 2026 15:31:51 +0000 Subject: [PATCH 2/2] chore(release): prepare 0.4.1 hotfix changelog --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8033422..2dfeaa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] +## [0.4.1] - 2026-02-23 + +### Added +- Regression tests for GPU parsing in `slurm_backend`, including realistic `scontrol show node -o` fixture coverage and obfuscated `squeue` job cases. +- CI workflow running tests on pull requests and pushes to `main`. + +### Fixed +- Correct GPU parsing for typed GRES values such as `gpu:h100:1` and `gres/gpu:h100:1` so model digits are not misread as GPU counts. +- Preserve per-node GPU semantics in job parsing (`per-node * nodes`) while keeping `gpu_total`/`total_gpu` as explicit totals. + ## [0.4.0] - 2026-02-23 ### Added @@ -62,7 +72,8 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ### Added - Initial `smon` release. -[Unreleased]: https://github.com/RuHae/smon/compare/v0.4.0...HEAD +[Unreleased]: https://github.com/RuHae/smon/compare/v0.4.1...HEAD +[0.4.1]: https://github.com/RuHae/smon/compare/v0.4.0...v0.4.1 [0.4.0]: https://github.com/RuHae/smon/compare/v0.3.0...v0.4.0 [0.3.0]: https://github.com/RuHae/smon/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/RuHae/smon/compare/v0.1.0...v0.2.0