# MPI Operator 版本:同域 2 节点 MNNVL NCCL 测试 # 前置条件: # 1. MPI Operator v0.8.0 已安装 # 2. ComputeDomain 已创建(sd-compute-domain-channel 存在) # 3. Worker 节点已标记 topology=same-domain --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-mpi-sd spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices count: 4 --- apiVersion: kubeflow.org/v2beta1 kind: MPIJob metadata: name: nccl-mpi-sd spec: slotsPerWorker: 4 runPolicy: ttlSecondsAfterFinished: 600 cleanPodPolicy: Running sshAuthMountPath: /root/.ssh launcherCreationPolicy: WaitForWorkersReady mpiReplicaSpecs: Launcher: replicas: 1 template: spec: nodeSelector: topology: same-domain initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true volumeMounts: - name: gib mountPath: /target/gib containers: - name: launcher image: nvcr.io/nvidia/pytorch:26.05-py3 imagePullPolicy: IfNotPresent env: - name: LD_LIBRARY_PATH value: /usr/local/gib/lib64:/usr/local/nvidia/lib64 command: - bash - -c - | set -e cd /tmp && git clone --depth 1 https://github.com/NVIDIA/nccl-tests.git cd nccl-tests && make -j$(nproc) MPI=1 \ MPI_HOME=/usr/local/mpi \ NCCL_HOME=/usr/local/gib \ CUDA_HOME=/usr/local/cuda source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true /usr/local/mpi/bin/mpirun --allow-run-as-root \ -np 8 -npernode 4 \ -x LD_LIBRARY_PATH=/usr/local/gib/lib64:/usr/local/nvidia/lib64 \ -x NCCL_NET_PLUGIN=/usr/local/gib/lib64/libnccl-net.so \ -x NCCL_ENV_PLUGIN=gcp \ -x NCCL_MNNVL_ENABLE=2 \ -x NCCL_CUMEM_ENABLE=1 \ --mca plm_rsh_args '-o StrictHostKeyChecking=no' \ /tmp/nccl-tests/build/all_reduce_perf -b 512M -e 8G -f 2 -g 1 volumeMounts: - name: gib mountPath: /usr/local/gib resources: requests: cpu: "4" memory: 16Gi limits: cpu: "4" memory: 16Gi volumes: - name: gib emptyDir: {} Worker: replicas: 2 template: spec: nodeSelector: topology: same-domain resourceClaims: - name: compute-domain-channel resourceClaimTemplateName: sd-compute-domain-channel - name: rdma-nics resourceClaimTemplateName: rdma-nics-mpi-sd initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true volumeMounts: - name: gib mountPath: /target/gib containers: - name: worker image: nvcr.io/nvidia/pytorch:26.05-py3 imagePullPolicy: IfNotPresent securityContext: privileged: true env: - name: LD_LIBRARY_PATH value: /usr/local/gib/lib64:/usr/local/nvidia/lib64 - name: NCCL_NET_PLUGIN value: /usr/local/gib/lib64/libnccl-net.so - name: NCCL_ENV_PLUGIN value: gcp command: - bash - -c - | apt-get update -qq && apt-get install -y -qq openssh-server > /dev/null 2>&1 mkdir -p /run/sshd printf 'Port 22\nPermitRootLogin yes\nPubkeyAuthentication yes\nAuthorizedKeysFile /root/.ssh/authorized_keys\nStrictModes no\nPasswordAuthentication no\nUsePAM no\nPidFile /run/sshd/sshd.pid\n' > /etc/ssh/mpi_sshd_config ssh-keygen -A cd /tmp && git clone --depth 1 https://github.com/NVIDIA/nccl-tests.git cd nccl-tests && make -j$(nproc) MPI=1 \ MPI_HOME=/usr/local/mpi \ NCCL_HOME=/usr/local/gib \ CUDA_HOME=/usr/local/cuda /usr/sbin/sshd -De -f /etc/ssh/mpi_sshd_config volumeMounts: - name: gib mountPath: /usr/local/gib - name: dshm mountPath: /dev/shm resources: requests: cpu: "96" memory: 800Gi limits: cpu: "96" memory: 800Gi nvidia.com/gpu: 4 claims: - name: compute-domain-channel - name: rdma-nics volumes: - name: gib emptyDir: {} - name: dshm emptyDir: medium: Memory sizeLimit: 250Gi