# Megatron-LM Cross-Domain Pod — k8s 1.34.1 + DRANET (无 ComputeDomain) # 跨域训练的第二个节点,不使用 MNNVL(域间通过 RDMA 通信) # 配合 k8s1341-megatron-train-dranet.yaml (Domain 0) 使用 --- apiVersion: v1 kind: Service metadata: name: mega-host-2 # ← 请根据实际环境修改:Service 名称 spec: selector: name: mega-host-2 clusterIP: None --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-mega-host-2-cd spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: rdma-devices # ← 请根据实际环境修改:RDMA DeviceClass 名称 count: 4 --- apiVersion: v1 kind: Pod metadata: name: mega-host-2 # ← 请根据实际环境修改:Pod 名称 labels: name: mega-host-2 spec: hostname: mega-h2 tolerations: - operator: "Exists" nodeSelector: topology: cross-domain # ← 请根据实际环境修改:跨域节点选择标签 volumes: - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: gib emptyDir: {} - name: data hostPath: path: /mnt/stateful_partition # ← 请根据实际环境修改:Local SSD 挂载路径 - name: scratch-data hostPath: path: /mnt/stateful_partition/scratch-data # ← 请根据实际环境修改:Local SSD 临时数据目录 initContainers: - name: gib-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.2 # ← 请根据实际环境修改:GIB 插件镜像版本 command: ["/bin/sh", "-c"] args: - | set -e /scripts/container_entry.sh install --install-nccl cp -a /usr/local/gib/. /target/gib/ cp -a /usr/lib/aarch64-linux-gnu/libibverbs.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libmlx5.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/librdmacm.so* /target/gib/lib64/ cp -a /usr/lib/aarch64-linux-gnu/libibumad.so* /target/gib/lib64/ 2>/dev/null || true cp -a /usr/lib/aarch64-linux-gnu/libibmad.so* /target/gib/lib64/ 2>/dev/null || true cp -a /usr/lib/aarch64-linux-gnu/libibnetdisc.so* /target/gib/lib64/ 2>/dev/null || true mkdir -p /target/gib/lib64/libibverbs cp -a /usr/lib/aarch64-linux-gnu/libibverbs/libmlx5-rdmav34.so /target/gib/lib64/libibverbs/ 2>/dev/null || true volumeMounts: - name: gib mountPath: /target/gib containers: - image: nvcr.io/nvidia/pytorch:26.05-py3 # ← 请根据实际环境修改:容器镜像版本 name: pytorch resources: limits: nvidia.com/gpu: 4 # ← 请根据实际环境修改:每节点 GPU 数量 claims: - name: rdma-nics securityContext: privileged: true volumeMounts: - name: shared-memory mountPath: /dev/shm - name: gib mountPath: /usr/local/gib - name: data mountPath: /data mountPropagation: HostToContainer - name: scratch-data mountPath: /scratch-data env: - name: LD_LIBRARY_PATH value: "/usr/local/gib/lib64:/usr/local/nvidia/lib64" - name: NCCL_MNNVL_ENABLE value: "0" - name: NCCL_CUMEM_ENABLE value: "0" command: ["/bin/bash", "-c"] args: - | source /usr/local/gib/scripts/set_nccl_env.sh 2>/dev/null || true apt-get update -qq && apt-get install -y -qq openssh-server 2>/dev/null mkdir -p /run/sshd /root/.ssh echo "StrictHostKeyChecking no" > /root/.ssh/config ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N '' -q cat /root/.ssh/id_ed25519.pub >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys /usr/sbin/sshd -p 2222 if [ -f /etc/pip/constraint.txt ]; then sed -i 's/setuptools==78\.1\.0/setuptools>=78.1.0/' /etc/pip/constraint.txt sed -i 's/packaging==23\.2/packaging>=23.2/' /etc/pip/constraint.txt fi mkdir -p /scratch-data /data/benchmark_cache if [ ! -d /scratch-data/Megatron-LM ]; then git clone --depth 1 --branch core_r0.16.0 https://github.com/NVIDIA/Megatron-LM.git /scratch-data/Megatron-LM cd /scratch-data/Megatron-LM && pip install -e . 2>&1 | tail -3 fi echo "hostname: $(hostname)" echo "Ready" sleep infinity resourceClaims: - name: rdma-nics resourceClaimTemplateName: rdma-nics-mega-host-2-cd