# NCCL Test — Single Node 4 GPU (intra-node NVLink baseline) # k8s 1.34.1 — no DRANET or ComputeDomain needed for single-node # Image: pytorch:26.05-py3 # TODO: 待机器就绪后验证 --- apiVersion: v1 kind: Pod metadata: name: nccl-single-node # ← 请根据实际环境修改:Pod 名称 labels: name: nccl-single-node spec: tolerations: - operator: "Exists" nodeSelector: topology: same-domain # ← 请根据实际环境修改:节点选择标签 volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true cp -a /usr/lib/aarch64-linux-gnu/libibmad.so* /target/gib/lib64/ 2>/dev/null || true cp -a /usr/lib/aarch64-linux-gnu/libibnetdisc.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true echo "GIB v1.1.2 + rdma-core installed" volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: nccl-test resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true echo "=== Single Node NCCL Test ===" echo "GPUs: $(nvidia-smi -L 2>&1 | wc -l)" echo "--- all_reduce_perf 4 GPU ---" /usr/local/bin/all_reduce_perf -b 1M -e 8G -f 2 -g 4 2>&1 echo "--- all_gather_perf 4 GPU ---" /usr/local/bin/all_gather_perf -b 1M -e 4G -f 2 -g 4 2>&1 echo "=== Done ===" sleep infinity