# DeepEP v2 Test — k8s 1.34.1 + DRANET + ComputeDomain # Same-domain 2 nodes, 4 GPUs each, IMEX via ComputeDomain # Tests: intranode BF16/FP8 + internode Elastic EP (NCCL GIN) --- apiVersion: v1 kind: Service metadata: name: deepep-h1 # ← 请根据实际环境修改:Service 名称 spec: selector: name: deepep-h1 clusterIP: None --- apiVersion: v1 kind: Service metadata: name: deepep-h2 # ← 请根据实际环境修改:Service 名称 spec: selector: name: deepep-h2 clusterIP: None --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-deepep-h1 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices count: 4 --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-deepep-h2 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices count: 4 --- apiVersion: v1 kind: Pod metadata: name: deepep-h1 # ← 请根据实际环境修改:Pod 名称 labels: name: deepep-h1 spec: tolerations: - operator: "Exists" nodeSelector: topology: same-domain # ← 请根据实际环境修改:节点选择标签 volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true cp -a /usr/lib/aarch64-linux-gnu/libibmad.so* /target/gib/lib64/ 2>/dev/null || true cp -a /usr/lib/aarch64-linux-gnu/libibnetdisc.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true echo "GIB v1.1.2 + rdma-core installed" volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: deepep resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 claims: - name: compute-domain-channel - name: rdma-nics securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" - name: NCCL_MNNVL_ENABLE value: "2" - name: NCCL_CUMEM_ENABLE value: "1" - name: NVSHMEM_BOOTSTRAP value: "MPI" - name: NVSHMEM_BOOTSTRAP_MPI_PLUGIN value: "nvshmem_bootstrap_torch.so" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true apt-get update -qq && apt-get install -y -qq openssh-server 2>/dev/null mkdir -p /run/sshd /root/.ssh echo "StrictHostKeyChecking no" > /root/.ssh/config ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N '' -q cat /root/.ssh/id_ed25519.pub >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys /usr/sbin/sshd -p 2222 if [ -f /etc/pip/constraint.txt ]; then sed -i 's/setuptools==78\.1\.0/setuptools>=78.1.0/' /etc/pip/constraint.txt sed -i 's/packaging==23\.2/packaging>=23.2/' /etc/pip/constraint.txt fi pip install nvidia-nvshmem-cu12==3.4.5 2>&1 | tail -3 ln -sf /usr/local/gib/lib64/libnccl.so.2 /usr/local/gib/lib64/libnccl.so apt-get update -qq && apt-get install -y -qq git 2>/dev/null cd /tmp && git clone --depth 1 https://github.com/deepseek-ai/DeepEP.git cd /tmp/DeepEP NCCL_DIR=/usr/local/gib TORCH_CUDA_ARCH_LIST='10.0' python setup.py build_ext --inplace 2>&1 | tail -5 echo "IMEX: $(ls /dev/nvidia-caps-imex-channels/ 2>&1)" echo "Ready - hostname: $(hostname)" sleep infinity resourceClaims: - name: compute-domain-channel resourceClaimTemplateName: sd-compute-domain-channel # ← 请根据实际环境修改:ComputeDomain 名称 - name: rdma-nics resourceClaimTemplateName: rdma-nics-deepep-h1 --- apiVersion: v1 kind: Pod metadata: name: deepep-h2 # ← 请根据实际环境修改:Pod 名称 labels: name: deepep-h2 spec: tolerations: - operator: "Exists" nodeSelector: topology: same-domain # ← 请根据实际环境修改:节点选择标签 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: name operator: In values: - deepep-h1 topologyKey: kubernetes.io/hostname volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true cp -a /usr/lib/aarch64-linux-gnu/libibmad.so* /target/gib/lib64/ 2>/dev/null || true cp -a /usr/lib/aarch64-linux-gnu/libibnetdisc.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: deepep resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 claims: - name: compute-domain-channel - name: rdma-nics securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" - name: NCCL_MNNVL_ENABLE value: "2" - name: NCCL_CUMEM_ENABLE value: "1" - name: NVSHMEM_BOOTSTRAP value: "MPI" - name: NVSHMEM_BOOTSTRAP_MPI_PLUGIN value: "nvshmem_bootstrap_torch.so" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true apt-get update -qq && apt-get install -y -qq openssh-server 2>/dev/null mkdir -p /run/sshd /root/.ssh echo "StrictHostKeyChecking no" > /root/.ssh/config ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N '' -q cat /root/.ssh/id_ed25519.pub >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys /usr/sbin/sshd -p 2222 if [ -f /etc/pip/constraint.txt ]; then sed -i 's/setuptools==78\.1\.0/setuptools>=78.1.0/' /etc/pip/constraint.txt sed -i 's/packaging==23\.2/packaging>=23.2/' /etc/pip/constraint.txt fi pip install nvidia-nvshmem-cu12==3.4.5 2>&1 | tail -3 ln -sf /usr/local/gib/lib64/libnccl.so.2 /usr/local/gib/lib64/libnccl.so apt-get update -qq && apt-get install -y -qq git 2>/dev/null cd /tmp && git clone --depth 1 https://github.com/deepseek-ai/DeepEP.git cd /tmp/DeepEP NCCL_DIR=/usr/local/gib TORCH_CUDA_ARCH_LIST='10.0' python setup.py build_ext --inplace 2>&1 | tail -5 echo "Ready - hostname: $(hostname)" sleep infinity resourceClaims: - name: compute-domain-channel resourceClaimTemplateName: sd-compute-domain-channel # ← 请根据实际环境修改:ComputeDomain 名称 - name: rdma-nics resourceClaimTemplateName: rdma-nics-deepep-h2