# NCCL Test — 4 Node Mixed Domain (2 same-domain + 2 cross-domain) # k8s 1.34.1 + DRA GPU Driver + DRANET v1.3.0 + ComputeDomain # 16 GPUs total across 4 nodes # Same-domain (h1,h2): MNNVL via ComputeDomain IMEX channel # Cross-domain (h3,h4): Pure RDMA, no MNNVL --- apiVersion: v1 kind: Service metadata: name: nccl-mix-h1 # ← 请根据实际环境修改:Service 名称 spec: selector: name: nccl-mix-h1 clusterIP: None --- apiVersion: v1 kind: Service metadata: name: nccl-mix-h2 # ← 请根据实际环境修改:Service 名称 spec: selector: name: nccl-mix-h2 clusterIP: None --- apiVersion: v1 kind: Service metadata: name: nccl-mix-h3 # ← 请根据实际环境修改:Service 名称 spec: selector: name: nccl-mix-h3 clusterIP: None --- apiVersion: v1 kind: Service metadata: name: nccl-mix-h4 # ← 请根据实际环境修改:Service 名称 spec: selector: name: nccl-mix-h4 clusterIP: None --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-mix-h1 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices count: 4 --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-mix-h2 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices count: 4 --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-mix-h3 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices count: 4 --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-mix-h4 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices count: 4 --- # Pod 1: same-domain node (with ComputeDomain IMEX channel for MNNVL) apiVersion: v1 kind: Pod metadata: name: nccl-mix-h1 # ← 请根据实际环境修改:Pod 名称 labels: name: nccl-mix-h1 spec: tolerations: - operator: "Exists" nodeSelector: topology: same-domain # ← 请根据实际环境修改:节点选择标签 volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: nccl-test resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 claims: - name: compute-domain-channel - name: rdma-nics securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" - name: NCCL_MNNVL_ENABLE value: "2" - name: NCCL_CUMEM_ENABLE value: "1" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true apt-get update -qq && apt-get install -y -qq openssh-server 2>/dev/null mkdir -p /run/sshd /root/.ssh echo "StrictHostKeyChecking no" > /root/.ssh/config ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N '' -q cat /root/.ssh/id_ed25519.pub >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys /usr/sbin/sshd -p 2222 echo "Ready - hostname: $(hostname)" sleep infinity resourceClaims: - name: compute-domain-channel resourceClaimTemplateName: sd-compute-domain-channel # ← 请根据实际环境修改:ComputeDomain 名称 - name: rdma-nics resourceClaimTemplateName: rdma-nics-mix-h1 --- # Pod 2: same-domain node (with ComputeDomain IMEX channel for MNNVL) apiVersion: v1 kind: Pod metadata: name: nccl-mix-h2 # ← 请根据实际环境修改:Pod 名称 labels: name: nccl-mix-h2 spec: tolerations: - operator: "Exists" nodeSelector: topology: same-domain # ← 请根据实际环境修改:节点选择标签 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: name operator: In values: - nccl-mix-h1 topologyKey: kubernetes.io/hostname volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: nccl-test resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 claims: - name: compute-domain-channel - name: rdma-nics securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" - name: NCCL_MNNVL_ENABLE value: "2" - name: NCCL_CUMEM_ENABLE value: "1" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true apt-get update -qq && apt-get install -y -qq openssh-server 2>/dev/null mkdir -p /run/sshd /root/.ssh echo "StrictHostKeyChecking no" > /root/.ssh/config ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N '' -q cat /root/.ssh/id_ed25519.pub >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys /usr/sbin/sshd -p 2222 echo "Ready - hostname: $(hostname)" sleep infinity resourceClaims: - name: compute-domain-channel resourceClaimTemplateName: sd-compute-domain-channel # ← 请根据实际环境修改:ComputeDomain 名称 - name: rdma-nics resourceClaimTemplateName: rdma-nics-mix-h2 --- # Pod 3: cross-domain node (RDMA only, no MNNVL) apiVersion: v1 kind: Pod metadata: name: nccl-mix-h3 # ← 请根据实际环境修改:Pod 名称 labels: name: nccl-mix-h3 spec: tolerations: - operator: "Exists" nodeSelector: topology: cross-domain # ← 请根据实际环境修改:节点选择标签 volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: nccl-test resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 claims: - name: rdma-nics securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" - name: NCCL_MNNVL_ENABLE value: "0" - name: NCCL_CUMEM_ENABLE value: "1" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true apt-get update -qq && apt-get install -y -qq openssh-server 2>/dev/null mkdir -p /run/sshd /root/.ssh echo "StrictHostKeyChecking no" > /root/.ssh/config ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N '' -q cat /root/.ssh/id_ed25519.pub >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys /usr/sbin/sshd -p 2222 echo "Ready - hostname: $(hostname)" sleep infinity resourceClaims: - name: rdma-nics resourceClaimTemplateName: rdma-nics-mix-h3 --- # Pod 4: cross-domain node (RDMA only, no MNNVL) apiVersion: v1 kind: Pod metadata: name: nccl-mix-h4 # ← 请根据实际环境修改:Pod 名称 labels: name: nccl-mix-h4 spec: tolerations: - operator: "Exists" nodeSelector: topology: cross-domain # ← 请根据实际环境修改:节点选择标签 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: name operator: In values: - nccl-mix-h3 topologyKey: kubernetes.io/hostname volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: nccl-test resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 claims: - name: rdma-nics securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" - name: NCCL_MNNVL_ENABLE value: "0" - name: NCCL_CUMEM_ENABLE value: "1" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true apt-get update -qq && apt-get install -y -qq openssh-server 2>/dev/null mkdir -p /run/sshd /root/.ssh echo "StrictHostKeyChecking no" > /root/.ssh/config ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N '' -q cat /root/.ssh/id_ed25519.pub >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys /usr/sbin/sshd -p 2222 echo "Ready - hostname: $(hostname)" sleep infinity resourceClaims: - name: rdma-nics resourceClaimTemplateName: rdma-nics-mix-h4