# NCCL Test — Same Domain 2 nodes with ComputeDomain + DRANET # k8s 1.34.1 + DRA GPU Driver v25.12.0 + DRANET v1.3.0 # Uses existing sd-compute-domain for IMEX channel # Image: pytorch:26.05-py3 --- apiVersion: v1 kind: Service metadata: name: nccl-sd-h1 # ← 请根据实际环境修改:Service 名称 spec: selector: name: nccl-sd-h1 clusterIP: None --- apiVersion: v1 kind: Service metadata: name: nccl-sd-h2 # ← 请根据实际环境修改:Service 名称 spec: selector: name: nccl-sd-h2 clusterIP: None --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-sd-h1 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices count: 4 --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-sd-h2 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices count: 4 --- apiVersion: v1 kind: Pod metadata: name: nccl-sd-h1 # ← 请根据实际环境修改:Pod 名称 labels: name: nccl-sd-h1 spec: tolerations: - operator: "Exists" nodeSelector: topology: same-domain # ← 请根据实际环境修改:节点选择标签 volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true echo "GIB v1.1.2 + rdma-core installed" volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: nccl-test resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 claims: - name: compute-domain-channel - name: rdma-nics securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" - name: NCCL_MNNVL_ENABLE value: "2" - name: NCCL_CUMEM_ENABLE value: "1" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true apt-get update -qq && apt-get install -y -qq openssh-server 2>/dev/null mkdir -p /run/sshd /root/.ssh echo "StrictHostKeyChecking no" > /root/.ssh/config ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N '' -q cat /root/.ssh/id_ed25519.pub >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys /usr/sbin/sshd -p 2222 echo "IMEX: $(ls /dev/nvidia-caps-imex-channels/ 2>&1)" echo "GPUs: $(nvidia-smi -L 2>&1 | wc -l)" echo "Ready - hostname: $(hostname)" sleep infinity resourceClaims: - name: compute-domain-channel resourceClaimTemplateName: sd-compute-domain-channel # ← 请根据实际环境修改:ComputeDomain 名称 - name: rdma-nics resourceClaimTemplateName: rdma-nics-sd-h1 --- apiVersion: v1 kind: Pod metadata: name: nccl-sd-h2 # ← 请根据实际环境修改:Pod 名称 labels: name: nccl-sd-h2 spec: tolerations: - operator: "Exists" nodeSelector: topology: same-domain # ← 请根据实际环境修改:节点选择标签 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: name operator: In values: - nccl-sd-h1 topologyKey: kubernetes.io/hostname volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: nccl-test resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 claims: - name: compute-domain-channel - name: rdma-nics securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" - name: NCCL_MNNVL_ENABLE value: "2" - name: NCCL_CUMEM_ENABLE value: "1" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true apt-get update -qq && apt-get install -y -qq openssh-server 2>/dev/null mkdir -p /run/sshd /root/.ssh echo "StrictHostKeyChecking no" > /root/.ssh/config ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N '' -q cat /root/.ssh/id_ed25519.pub >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys /usr/sbin/sshd -p 2222 echo "IMEX: $(ls /dev/nvidia-caps-imex-channels/ 2>&1)" echo "Ready - hostname: $(hostname)" sleep infinity resourceClaims: - name: compute-domain-channel resourceClaimTemplateName: sd-compute-domain-channel # ← 请根据实际环境修改:ComputeDomain 名称 - name: rdma-nics resourceClaimTemplateName: rdma-nics-sd-h2