Skip to content

Commit 0cbd865

Browse files
rgarciargarcia
committed
Enable kernel modules and DRM for NVIDIA GPU passthrough
- Enable CONFIG_MODULES for loadable module support - Enable CONFIG_DRM and CONFIG_DRM_BOCHS (which selects DRM_TTM) - Add CI steps to build NVIDIA open-gpu-kernel-modules - Download NVIDIA driver and extract GSP firmware for H100/modern GPUs - Package kernel modules + firmware as nvidia-modules-x86_64.tar.gz - Package userspace driver libraries as nvidia-driver-libs-x86_64.tar.gz (libcuda.so, libnvidia-ml.so, nvidia-smi, etc. for container injection)
1 parent 0303f29 commit 0cbd865

4 files changed

Lines changed: 108 additions & 6 deletions

File tree

‎.github/workflows/release.yaml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,81 @@ jobs:
2828
run: make ch_defconfig
2929
- name: Build kernel (x86-64)
3030
run: CFLAGS="-Wa,-mx86-used-note=no" make bzImage -j `nproc`
31+
- name: Build kernel modules (x86-64)
32+
run: make modules -j `nproc`
33+
- name: Clone NVIDIA open-gpu-kernel-modules
34+
run: git clone --depth 1 --branch 570.86.16 https://github.com/NVIDIA/open-gpu-kernel-modules.git
35+
- name: Build NVIDIA modules (x86-64)
36+
run: |
37+
cd open-gpu-kernel-modules
38+
make modules KERNEL_UNAME=$(cd .. && make -s kernelrelease) SYSSRC=$(pwd)/.. -j $(nproc)
39+
- name: Download NVIDIA driver for firmware
40+
run: |
41+
DRIVER_VERSION=570.86.16
42+
wget -q https://download.nvidia.com/XFree86/Linux-x86_64/${DRIVER_VERSION}/NVIDIA-Linux-x86_64-${DRIVER_VERSION}.run
43+
chmod +x NVIDIA-Linux-x86_64-${DRIVER_VERSION}.run
44+
./NVIDIA-Linux-x86_64-${DRIVER_VERSION}.run --extract-only --target nvidia-driver
45+
- name: Package NVIDIA modules (x86-64)
46+
run: |
47+
KVER=$(make -s kernelrelease)
48+
DRIVER_VERSION=570.86.16
49+
mkdir -p nvidia-modules/lib/modules/$KVER/kernel/drivers/gpu
50+
mkdir -p nvidia-modules/lib/firmware/nvidia/${DRIVER_VERSION}
51+
cp open-gpu-kernel-modules/kernel-open/*.ko nvidia-modules/lib/modules/$KVER/kernel/drivers/gpu/
52+
cp nvidia-driver/firmware/*.bin nvidia-modules/lib/firmware/nvidia/${DRIVER_VERSION}/
53+
tar czf nvidia-modules-x86_64.tar.gz -C nvidia-modules .
54+
# ============================================================
55+
# NVIDIA DRIVER USERSPACE LIBRARIES
56+
# ============================================================
57+
# These libraries are injected into containers at VM boot time by hypeman,
58+
# eliminating the need for containers to bundle matching NVIDIA drivers.
59+
#
60+
# When upgrading the driver version:
61+
# 1. Check NVIDIA release notes: https://download.nvidia.com/XFree86/Linux-x86_64/
62+
# 2. Ensure compatibility with the kernel version being built
63+
# 3. Update DRIVER_VERSION in ALL places in this file (search for 570.86.16)
64+
# 4. Update NvidiaDriverVersion in hypeman/lib/system/versions.go
65+
# 5. Test GPU passthrough thoroughly before deploying
66+
# ============================================================
67+
- name: Package NVIDIA driver libraries (x86-64)
68+
run: |
69+
DRIVER_VERSION=570.86.16
70+
71+
mkdir -p nvidia-driver-libs/usr/lib/nvidia
72+
mkdir -p nvidia-driver-libs/usr/bin
73+
74+
# Essential libraries for CUDA inference (minimal set to keep initrd small)
75+
# libcuda.so - CUDA driver API (required for all GPU compute) ~68MB
76+
cp nvidia-driver/libcuda.so.${DRIVER_VERSION} nvidia-driver-libs/usr/lib/nvidia/
77+
# libnvidia-ml.so - NVML for nvidia-smi and GPU monitoring ~2MB
78+
cp nvidia-driver/libnvidia-ml.so.${DRIVER_VERSION} nvidia-driver-libs/usr/lib/nvidia/
79+
# libnvidia-ptxjitcompiler.so - PTX JIT compilation ~37MB
80+
cp nvidia-driver/libnvidia-ptxjitcompiler.so.${DRIVER_VERSION} nvidia-driver-libs/usr/lib/nvidia/
81+
82+
# Small utility libraries
83+
cp nvidia-driver/libnvidia-allocator.so.${DRIVER_VERSION} nvidia-driver-libs/usr/lib/nvidia/
84+
cp nvidia-driver/libnvidia-cfg.so.${DRIVER_VERSION} nvidia-driver-libs/usr/lib/nvidia/
85+
86+
# Note: The following large libraries are NOT included to keep initrd small:
87+
# - libnvidia-nvvm.so (~79MB) - Only needed for CUDA runtime compilation
88+
# - libnvidia-gpucomp.so (~61MB) - Only needed for some compute workloads
89+
# - libnvidia-opencl.so (~63MB) - Only needed for OpenCL (not CUDA)
90+
# - libnvcuvid.so (~16MB) - Only needed for video decoding
91+
# - libnvidia-encode.so - Only needed for video encoding
92+
# Containers requiring these can install them directly.
93+
94+
# Binaries
95+
cp nvidia-driver/nvidia-smi nvidia-driver-libs/usr/bin/
96+
cp nvidia-driver/nvidia-modprobe nvidia-driver-libs/usr/bin/
97+
98+
# Version file for runtime verification
99+
echo "${DRIVER_VERSION}" > nvidia-driver-libs/usr/lib/nvidia/version
100+
101+
tar czf nvidia-driver-libs-x86_64.tar.gz -C nvidia-driver-libs .
102+
103+
echo "Driver libraries packaged:"
104+
ls -lah nvidia-driver-libs/usr/lib/nvidia/
105+
ls -lah nvidia-driver-libs/usr/bin/
31106
- name: Create release
32107
id: create_release
33108
uses: actions/create-release@v1
@@ -55,6 +130,24 @@ jobs:
55130
asset_path: vmlinux
56131
asset_name: vmlinux-x86_64
57132
asset_content_type: application/octet-stream
133+
- name: Upload NVIDIA modules for x86_64
134+
uses: actions/upload-release-asset@v1
135+
env:
136+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
137+
with:
138+
upload_url: ${{ steps.create_release.outputs.upload_url }}
139+
asset_path: nvidia-modules-x86_64.tar.gz
140+
asset_name: nvidia-modules-x86_64.tar.gz
141+
asset_content_type: application/gzip
142+
- name: Upload NVIDIA driver libraries for x86_64
143+
uses: actions/upload-release-asset@v1
144+
env:
145+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
146+
with:
147+
upload_url: ${{ steps.create_release.outputs.upload_url }}
148+
asset_path: nvidia-driver-libs-x86_64.tar.gz
149+
asset_name: nvidia-driver-libs-x86_64.tar.gz
150+
asset_content_type: application/gzip
58151
- name: Upload Image.gz for aarch64
59152
uses: actions/upload-release-asset@v1
60153
env:

‎arch/arm64/configs/ch_defconfig‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -794,7 +794,9 @@ CONFIG_FUNCTION_ALIGNMENT=4
794794
# end of General architecture-dependent options
795795

796796
CONFIG_RT_MUTEXES=y
797-
# CONFIG_MODULES is not set
797+
CONFIG_MODULES=y
798+
CONFIG_MODULE_UNLOAD=y
799+
CONFIG_MODVERSIONS=y
798800
CONFIG_BLOCK=y
799801
CONFIG_BLOCK_LEGACY_AUTOLOAD=y
800802
CONFIG_BLK_CGROUP_RWSTAT=y
@@ -2002,7 +2004,8 @@ CONFIG_BCMA_POSSIBLE=y
20022004
#
20032005
CONFIG_VIDEO=y
20042006
# CONFIG_AUXDISPLAY is not set
2005-
# CONFIG_DRM is not set
2007+
CONFIG_DRM=y
2008+
CONFIG_DRM_BOCHS=y
20062009

20072010
#
20082011
# Frame buffer Devices

‎arch/riscv/configs/ch_defconfig‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -655,7 +655,9 @@ CONFIG_FUNCTION_ALIGNMENT=0
655655
# end of General architecture-dependent options
656656

657657
CONFIG_RT_MUTEXES=y
658-
# CONFIG_MODULES is not set
658+
CONFIG_MODULES=y
659+
CONFIG_MODULE_UNLOAD=y
660+
CONFIG_MODVERSIONS=y
659661
CONFIG_BLOCK=y
660662
CONFIG_BLOCK_LEGACY_AUTOLOAD=y
661663
CONFIG_BLK_CGROUP_RWSTAT=y
@@ -1769,7 +1771,8 @@ CONFIG_BCMA_POSSIBLE=y
17691771
#
17701772
CONFIG_VIDEO=y
17711773
# CONFIG_AUXDISPLAY is not set
1772-
# CONFIG_DRM is not set
1774+
CONFIG_DRM=y
1775+
CONFIG_DRM_BOCHS=y
17731776

17741777
#
17751778
# Frame buffer Devices

‎arch/x86/configs/ch_defconfig‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -857,7 +857,9 @@ CONFIG_CC_HAS_SANE_FUNCTION_ALIGNMENT=y
857857
# end of General architecture-dependent options
858858

859859
CONFIG_RT_MUTEXES=y
860-
# CONFIG_MODULES is not set
860+
CONFIG_MODULES=y
861+
CONFIG_MODULE_UNLOAD=y
862+
CONFIG_MODVERSIONS=y
861863
CONFIG_BLOCK=y
862864
CONFIG_BLOCK_LEGACY_AUTOLOAD=y
863865
CONFIG_BLK_CGROUP_RWSTAT=y
@@ -2024,7 +2026,8 @@ CONFIG_VIDEO=y
20242026
# CONFIG_AUXDISPLAY is not set
20252027
# CONFIG_AGP is not set
20262028
# CONFIG_VGA_SWITCHEROO is not set
2027-
# CONFIG_DRM is not set
2029+
CONFIG_DRM=y
2030+
CONFIG_DRM_BOCHS=y
20282031

20292032
#
20302033
# Frame buffer Devices

0 commit comments

Comments
 (0)