feat(nvidia): Vulkan vGPU support — memory partitioning via vulkan-layer + manifest auto-mount + admission webhook env injection

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "libvgpu"]
 	path = libvgpu
-	url = https://github.com/Project-HAMi/HAMi-core.git
-	branch = main
+	url = https://github.com/xiilab/HAMi-core.git
+	branch = vulkan-layer

charts/hami/templates/scheduler/webhook.yaml

@@ -29,10 +29,17 @@ webhooks:
         {{- toYaml .Values.scheduler.admissionWebhook.namespaceSelector.matchLabels | nindent 8 }}
       {{- end }}
       matchExpressions:
+      {{- if eq (.Values.scheduler.admissionWebhook.namespaceSelector.mode | default "opt-out") "opt-in" }}
+      - key: hami.io/vgpu
+        operator: In
+        values:
+        - enabled
+      {{- else }}
       - key: hami.io/webhook
         operator: NotIn
         values:
         - ignore
+      {{- end }}
       {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }}
       - key: kubernetes.io/metadata.name
         operator: NotIn

charts/hami/values.yaml

@@ -172,19 +172,19 @@ scheduler:
       # - default
       # - kube-system
       # - istio-system
-    # namespaceSelector controls which namespaces the webhook will be applied to.
-    # The default matchExpressions exclude namespaces with label "hami.io/webhook: ignore".
-    # You can add additional matchLabels or matchExpressions to further filter namespaces.
+    # namespaceSelector controls which namespaces the webhook will apply to.
+    # mode:
+    #   "opt-out" (legacy default): apply to all namespaces except those labeled
+    #              hami.io/webhook=ignore. Suitable when most workloads need vGPU
+    #              isolation and a small number opt out.
+    #   "opt-in"  (recommended for clusters with NVIDIA Omniverse / Isaac Sim
+    #              workloads that conflict with HAMi-core hooks): apply ONLY to
+    #              namespaces labeled hami.io/vgpu=enabled. Other namespaces see
+    #              no mutation, no LD_PRELOAD inject, no implicit Vulkan layer.
     namespaceSelector:
-      matchLabels:
-        # app.kubernetes.io/part-of: kubeflow-profile
+      mode: opt-in
+      matchLabels: {}
       matchExpressions: []
-      # Example: exclude namespaces with specific labels
-      # - key: environment
-      #   operator: In
-      #   values:
-      #   - development
-      #   - staging
     # objectSelector controls which pods the webhook will be applied to.
     # The default matchExpressions exclude pods with label "hami.io/webhook: ignore".
     # You can add additional matchLabels or matchExpressions to further filter pods.

cluster/runtime/snapshot-2026-04-28/hami-vulkan-manifest-cm.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+data:
+  hami.json: |
+    {
+        "file_format_version": "1.0.0",
+        "layer": {
+            "name": "VK_LAYER_HAMI_vgpu",
+            "type": "GLOBAL",
+            "library_path": "/usr/local/vgpu/libvgpu.so",
+            "api_version": "1.3.0",
+            "implementation_version": "1",
+            "description": "HAMi Vulkan vGPU memory partitioning layer",
+            "enable_environment": {
+                "HAMI_VULKAN_ENABLE": "1"
+            }
+        }
+    }
+kind: ConfigMap
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"v1","data":{"hami.json":"{\n    \"file_format_version\": \"1.0.0\",\n    \"layer\": {\n        \"name\": \"VK_LAYER_HAMI_vgpu\",\n        \"type\": \"GLOBAL\",\n        \"library_path\": \"/usr/local/vgpu/libvgpu.so\",\n        \"api_version\": \"1.3.0\",\n        \"implementation_version\": \"1\",\n        \"description\": \"HAMi Vulkan vGPU memory partitioning layer\",\n        \"enable_environment\": {\n            \"HAMI_VULKAN_ENABLE\": \"1\"\n        }\n    }\n}\n"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"hami-vulkan-manifest","namespace":"kube-system"}}
+  creationTimestamp: "2026-04-27T02:17:50Z"
+  name: hami-vulkan-manifest
+  namespace: kube-system
+  resourceVersion: "20078116"
+  uid: cf25a104-6177-43ee-8a64-0483ef5901fa

cluster/runtime/snapshot-2026-04-28/hami-vulkan-manifest-installer-ds.yaml

@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  annotations:
+    deprecated.daemonset.template.generation: "2"
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"apps/v1","kind":"DaemonSet","metadata":{"annotations":{},"labels":{"app":"hami-vulkan-manifest-installer"},"name":"hami-vulkan-manifest-installer","namespace":"kube-system"},"spec":{"selector":{"matchLabels":{"app":"hami-vulkan-manifest-installer"}},"template":{"metadata":{"labels":{"app":"hami-vulkan-manifest-installer"}},"spec":{"containers":[{"command":["/bin/sh","-c","set -eu\nmkdir -p /host/usr/local/vgpu/vulkan/implicit_layer.d\ncp -f /manifest/hami.json \\\n      /host/usr/local/vgpu/vulkan/implicit_layer.d/hami.json\necho \"[hami-vulkan-manifest] installed at /usr/local/vgpu/vulkan/implicit_layer.d/hami.json\"\n# DaemonSet 이라 종료하지 않고 sleep — restart 루프 회피\nsleep infinity\n"],"image":"busybox:1.36","name":"installer","securityContext":{"runAsUser":0},"volumeMounts":[{"mountPath":"/manifest","name":"manifest","readOnly":true},{"mountPath":"/host/usr/local/vgpu","name":"host-vgpu"}]}],"hostPID":false,"nodeSelector":{"nvidia.com/gpu.present":"true"},"restartPolicy":"Always","tolerations":[{"operator":"Exists"}],"volumes":[{"configMap":{"name":"hami-vulkan-manifest"},"name":"manifest"},{"hostPath":{"path":"/usr/local/vgpu","type":"DirectoryOrCreate"},"name":"host-vgpu"}]}}}}
+  creationTimestamp: "2026-04-27T02:17:50Z"
+  generation: 2
+  labels:
+    app: hami-vulkan-manifest-installer
+  name: hami-vulkan-manifest-installer
+  namespace: kube-system
+  resourceVersion: "20897004"
+  uid: 6998669a-7bd7-463b-9564-94ad1a4f3feb
+spec:
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: hami-vulkan-manifest-installer
+  template:
+    metadata:
+      labels:
+        app: hami-vulkan-manifest-installer
+    spec:
+      containers:
+      - command:
+        - /bin/sh
+        - -c
+        - |
+          set -eu
+          mkdir -p /host/usr/local/vgpu/vulkan/implicit_layer.d
+          cp -f /manifest/hami.json \
+                /host/usr/local/vgpu/vulkan/implicit_layer.d/hami.json
+          echo "[hami-vulkan-manifest] installed at /usr/local/vgpu/vulkan/implicit_layer.d/hami.json"
+          # DaemonSet 이라 종료하지 않고 sleep — restart 루프 회피
+          sleep infinity
+        image: busybox:1.36
+        imagePullPolicy: IfNotPresent
+        name: installer
+        resources: {}
+        securityContext:
+          runAsUser: 0
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /manifest
+          name: manifest
+          readOnly: true
+        - mountPath: /host/usr/local/vgpu
+          name: host-vgpu
+      dnsPolicy: ClusterFirst
+      nodeSelector:
+        hami.io/disabled: "true"
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      tolerations:
+      - operator: Exists
+      volumes:
+      - configMap:
+          defaultMode: 420
+          name: hami-vulkan-manifest
+        name: manifest
+      - hostPath:
+          path: /usr/local/vgpu
+          type: DirectoryOrCreate
+        name: host-vgpu
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+status:
+  currentNumberScheduled: 0
+  desiredNumberScheduled: 0
+  numberMisscheduled: 0
+  numberReady: 0
+  observedGeneration: 2

cluster/runtime/snapshot-2026-04-28/hami-webhook-mutating.yaml

@@ -0,0 +1,51 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  annotations:
+    meta.helm.sh/release-name: hami-webhook
+    meta.helm.sh/release-namespace: hami-system
+  creationTimestamp: "2026-04-27T03:34:22Z"
+  generation: 2
+  labels:
+    app.kubernetes.io/managed-by: Helm
+  name: hami-webhook-webhook
+  resourceVersion: "20112390"
+  uid: 32e51184-bf13-41df-a382-1566671cc010
+webhooks:
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkakNDQVJ5Z0F3SUJBZ0lSQVBlREVjTWFneDYwOENEVWJYdUZXdjR3Q2dZSUtvWkl6ajBFQXdJd0R6RU4KTUFzR0ExVUVDaE1FYm1sc01UQWdGdzB5TmpBME1qY3dNakl5TlROYUdBOHlNVEkyTURRd016QXlNakkxTTFvdwpEekVOTUFzR0ExVUVDaE1FYm1sc01UQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJIc2dFbU95CnpQWllFTHEwMmFIQUIxRXI1MnE1dlE0Vi9qV2hTVmg2S0RIRUh6Y1JFNUdlQ29nMDBMbDBaYnlDZXZ1RmhETGcKdDBYZXlXOVJaTldMVDF1alZ6QlZNQTRHQTFVZER3RUIvd1FFQXdJQ0JEQVRCZ05WSFNVRUREQUtCZ2dyQmdFRgpCUWNEQVRBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJSOHk5ZXVCOFVIaVZ0bWtvQkh6VzVDCnZXRXJHekFLQmdncWhrak9QUVFEQWdOSUFEQkZBaUVBdUJ2eWVmekJoam5rclpYN0hKRzNURzVRZ2NzTVpvVC8KUmpPQWFhNEFrNHNDSUEvaGhyMndkODE2M2loNmEvWWtuTWdhOXRCeVpGQ2lxeXF3NjhVUjRaNVIKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+    service:
+      name: hami-webhook-scheduler
+      namespace: hami-system
+      path: /webhook
+      port: 443
+  failurePolicy: Ignore
+  matchPolicy: Equivalent
+  name: vgpu.hami.io
+  namespaceSelector:
+    matchExpressions:
+    - key: hami.io/webhook
+      operator: NotIn
+      values:
+      - ignore
+  objectSelector:
+    matchExpressions:
+    - key: hami.io/webhook
+      operator: NotIn
+      values:
+      - ignore
+  reinvocationPolicy: Never
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - pods
+    scope: '*'
+  sideEffects: None
+  timeoutSeconds: 10

cluster/runtime/snapshot-2026-04-28/volcano-device-plugin-ds.yaml

@@ -0,0 +1,180 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  annotations:
+    deprecated.daemonset.template.generation: "5"
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"apps/v1","kind":"DaemonSet","metadata":{"annotations":{},"name":"volcano-device-plugin","namespace":"kube-system"},"spec":{"selector":{"matchLabels":{"name":"volcano-device-plugin"}},"template":{"metadata":{"annotations":{"scheduler.alpha.kubernetes.io/critical-pod":""},"labels":{"name":"volcano-device-plugin"}},"spec":{"containers":[{"args":["--device-split-count=10"],"env":[{"name":"NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"HOOK_PATH","value":"/usr/local/vgpu"},{"name":"NVIDIA_VISIBLE_DEVICES","value":"all"},{"name":"NVIDIA_MIG_MONITOR_DEVICES","value":"all"},{"name":"NVIDIA_DRIVER_CAPABILITIES","value":"utility"}],"image":"10.61.3.124:30002/library/volcano-vgpu-device-plugin:vulkan-v1","lifecycle":{"postStart":{"exec":{"command":["/bin/sh","-c","cp -rf /k8s-vgpu/lib/nvidia/. /usr/local/vgpu/"]}}},"name":"volcano-device-plugin","securityContext":{"allowPrivilegeEscalation":true,"capabilities":{"add":["SYS_ADMIN"],"drop":["ALL"]},"privileged":true},"volumeMounts":[{"mountPath":"/config","name":"deviceconfig"},{"mountPath":"/var/lib/kubelet/device-plugins","name":"device-plugin"},{"mountPath":"/usr/local/vgpu","name":"lib"},{"mountPath":"/tmp","name":"hosttmp"}]},{"command":["/bin/bash","-c","volcano-vgpu-monitor"],"env":[{"name":"NVIDIA_VISIBLE_DEVICES","value":"all"},{"name":"NVIDIA_MIG_MONITOR_DEVICES","value":"all"},{"name":"HOOK_PATH","value":"/tmp/vgpu"},{"name":"NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}}],"image":"10.61.3.124:30002/library/volcano-vgpu-device-plugin:vulkan-v1","name":"monitor","securityContext":{"allowPrivilegeEscalation":true,"capabilities":{"add":["SYS_ADMIN"],"drop":["ALL"]},"privileged":true},"volumeMounts":[{"mountPath":"/run/docker","name":"dockers"},{"mountPath":"/run/containerd","name":"containerds"},{"mountPath":"/sysinfo","name":"sysinfo"},{"mountPath":"/hostvar","name":"hostvar"},{"mountPath":"/tmp","name":"hosttmp"}]}],"priorityClassName":"system-node-critical","serviceAccount":"volcano-device-plugin","tolerations":[{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoSchedule","key":"volcano.sh/gpu-memory","operator":"Exists"}],"volumes":[{"configMap":{"name":"volcano-vgpu-node-config"},"name":"deviceconfig"},{"hostPath":{"path":"/var/lib/kubelet/device-plugins","type":"Directory"},"name":"device-plugin"},{"hostPath":{"path":"/usr/local/vgpu","type":"DirectoryOrCreate"},"name":"lib"},{"hostPath":{"path":"/tmp","type":"DirectoryOrCreate"},"name":"hosttmp"},{"hostPath":{"path":"/run/docker","type":"DirectoryOrCreate"},"name":"dockers"},{"hostPath":{"path":"/run/containerd","type":"DirectoryOrCreate"},"name":"containerds"},{"hostPath":{"path":"/usr/bin","type":"Directory"},"name":"usrbin"},{"hostPath":{"path":"/sys","type":"Directory"},"name":"sysinfo"},{"hostPath":{"path":"/var","type":"Directory"},"name":"hostvar"}]}},"updateStrategy":{"type":"RollingUpdate"}}}
+  creationTimestamp: "2026-04-27T02:17:52Z"
+  generation: 5
+  name: volcano-device-plugin
+  namespace: kube-system
+  resourceVersion: "20250370"
+  uid: f0d77283-6b73-419e-8504-6d4965dde85a
+spec:
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      name: volcano-device-plugin
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/restartedAt: "2026-04-27T17:44:34+09:00"
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        name: volcano-device-plugin
+    spec:
+      containers:
+      - args:
+        - --device-split-count=10
+        - --gpu-memory-factor=1024
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: HOOK_PATH
+          value: /usr/local/vgpu
+        - name: NVIDIA_VISIBLE_DEVICES
+          value: all
+        - name: NVIDIA_MIG_MONITOR_DEVICES
+          value: all
+        - name: NVIDIA_DRIVER_CAPABILITIES
+          value: utility
+        image: 10.61.3.124:30002/library/volcano-vgpu-device-plugin:vulkan-v1
+        imagePullPolicy: IfNotPresent
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - cp -rf /k8s-vgpu/lib/nvidia/. /usr/local/vgpu/
+        name: volcano-device-plugin
+        resources: {}
+        securityContext:
+          allowPrivilegeEscalation: true
+          capabilities:
+            add:
+            - SYS_ADMIN
+            drop:
+            - ALL
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /config
+          name: deviceconfig
+        - mountPath: /var/lib/kubelet/device-plugins
+          name: device-plugin
+        - mountPath: /usr/local/vgpu
+          name: lib
+        - mountPath: /tmp
+          name: hosttmp
+      - command:
+        - /bin/bash
+        - -c
+        - volcano-vgpu-monitor
+        env:
+        - name: NVIDIA_VISIBLE_DEVICES
+          value: all
+        - name: NVIDIA_MIG_MONITOR_DEVICES
+          value: all
+        - name: HOOK_PATH
+          value: /tmp/vgpu
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        image: 10.61.3.124:30002/library/volcano-vgpu-device-plugin:vulkan-v1
+        imagePullPolicy: IfNotPresent
+        name: monitor
+        resources: {}
+        securityContext:
+          allowPrivilegeEscalation: true
+          capabilities:
+            add:
+            - SYS_ADMIN
+            drop:
+            - ALL
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /run/docker
+          name: dockers
+        - mountPath: /run/containerd
+          name: containerds
+        - mountPath: /sysinfo
+          name: sysinfo
+        - mountPath: /hostvar
+          name: hostvar
+        - mountPath: /tmp
+          name: hosttmp
+      dnsPolicy: ClusterFirst
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      runtimeClassName: nvidia
+      schedulerName: default-scheduler
+      securityContext: {}
+      serviceAccount: volcano-device-plugin
+      serviceAccountName: volcano-device-plugin
+      terminationGracePeriodSeconds: 30
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoSchedule
+        key: volcano.sh/gpu-memory
+        operator: Exists
+      volumes:
+      - configMap:
+          defaultMode: 420
+          name: volcano-vgpu-node-config
+        name: deviceconfig
+      - hostPath:
+          path: /var/lib/kubelet/device-plugins
+          type: Directory
+        name: device-plugin
+      - hostPath:
+          path: /usr/local/vgpu
+          type: DirectoryOrCreate
+        name: lib
+      - hostPath:
+          path: /tmp
+          type: DirectoryOrCreate
+        name: hosttmp
+      - hostPath:
+          path: /run/docker
+          type: DirectoryOrCreate
+        name: dockers
+      - hostPath:
+          path: /run/containerd
+          type: DirectoryOrCreate
+        name: containerds
+      - hostPath:
+          path: /usr/bin
+          type: Directory
+        name: usrbin
+      - hostPath:
+          path: /sys
+          type: Directory
+        name: sysinfo
+      - hostPath:
+          path: /var
+          type: Directory
+        name: hostvar
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+status:
+  currentNumberScheduled: 1
+  desiredNumberScheduled: 1
+  numberAvailable: 1
+  numberMisscheduled: 0
+  numberReady: 1
+  observedGeneration: 5
+  updatedNumberScheduled: 1