apiVersion: v1 kind: Service metadata: name: cuda13-dra-1 spec: selector: name: cuda13-dra-1 clusterIP: None --- apiVersion: v1 kind: Service metadata: name: cuda13-dra-2 spec: selector: name: cuda13-dra-2 clusterIP: None --- apiVersion: resource.nvidia.com/v1beta1 kind: ComputeDomain metadata: name: cuda13-dra-compute-domain spec: numNodes: 2 channel: resourceClaimTemplate: name: cuda13-dra-compute-domain-channel --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-cuda13-dra-1 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: mrdma.google.com count: 4 --- apiVersion: resource.k8s.io/v1 kind: ResourceClaimTemplate metadata: name: rdma-nics-cuda13-dra-2 spec: spec: devices: requests: - name: rdma-nics exactly: deviceClassName: mrdma.google.com count: 4 --- apiVersion: v1 kind: Pod metadata: name: cuda13-dra-1 labels: name: cuda13-dra-1 annotations: gke-gcsfuse/volumes: "true" gke-gcsfuse/cpu-limit: "0" gke-gcsfuse/memory-limit: "0" gke-gcsfuse/ephemeral-storage-limit: "0" spec: tolerations: - key: nvidia.com/gpu operator: Exists effect: NoSchedule - key: kubernetes.io/arch operator: Exists effect: NoSchedule affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: cloud.google.com/gke-accelerator operator: In values: - nvidia-gb200 - key: kubernetes.io/arch operator: In values: - arm64 - key: cloud.google.com/gke-nodepool operator: In values: - a4x-gpu-pool-0 podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: name operator: In values: - gpu-1 - gpu-2 - cuda13-1 - cuda13-2 - cuda13-dra-2 topologyKey: kubernetes.io/hostname hostNetwork: false hostPID: false serviceAccount: workload-identity-k8s-sa volumes: - name: local-ssd hostPath: path: /mnt/stateful_partition/kube-ephemeral-ssd - name: library-dir-host hostPath: path: /home/kubernetes/bin/nvidia - name: gib hostPath: path: /home/kubernetes/bin/gib - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: data-volume persistentVolumeClaim: claimName: gcs-pvc containers: - image: nvcr.io/nvidia/pytorch:26.04-py3 name: pytorch resources: requests: cpu: 150m limits: nvidia.com/gpu: 4 claims: - name: compute-domain-channel - name: rdma-nics securityContext: privileged: true volumeMounts: - name: local-ssd mountPath: /scratch-data - name: library-dir-host mountPath: /usr/local/nvidia - name: gib mountPath: /usr/local/gib - name: shared-memory mountPath: /dev/shm - name: data-volume mountPath: /data env: - name: N_NODES value: "2" command: ["/bin/bash", "-c"] args: - | export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH} /sbin/ldconfig apt update -y apt install -y iputils-ping openssh-server iproute2 pciutils curl for i in $(seq 0 3); do nvidia-smi -i $i -pm ENABLED done mkdir -p /root/.ssh/ cp /data/a4x-keys/id_rsa /data/a4x-keys/id_rsa.pub /data/a4x-keys/authorized_keys /root/.ssh/ chmod 400 /root/.ssh/id_rsa chmod 600 /root/.ssh/authorized_keys sed -i 's/^#PermitRootLogin prohibit-password/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config service ssh restart echo "export LD_LIBRARY_PATH=/usr/local/gib/lib64:/usr/local/nvidia/lib64:\${LD_LIBRARY_PATH}" | tee -a /root/.bashrc > /dev/null export POSTFIX=$(hostname | cut -d . -f 2-) export WORKERS_BASENAME=$(hostname | cut -d . -f 1 | rev | cut -d - -f 2- | rev) for i in $(seq 1 ${N_NODES}); do OTHER=${WORKERS_BASENAME}-${i} until ssh -o StrictHostKeyChecking=no $OTHER hostname; do echo "Waiting for ${OTHER}..." sleep 10 done echo "${OTHER} slots=4" | tee -a /tmp/hostfile done sleep infinity initContainers: - name: nccl-plugin-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.0 imagePullPolicy: Always command: ["/bin/sh", "-c"] args: - | set -ex /scripts/container_entry.sh install --install-nccl cp -R /var/lib/gib/lib64/. /target/usr/local/gib/lib64 cp -R /var/lib/gib/. /target/usr/local/gib volumeMounts: - mountPath: /target/usr/local/gib name: gib - mountPath: /target/dev/shm name: shared-memory resourceClaims: - name: compute-domain-channel resourceClaimTemplateName: cuda13-dra-compute-domain-channel - name: rdma-nics resourceClaimTemplateName: rdma-nics-cuda13-dra-1 --- apiVersion: v1 kind: Pod metadata: name: cuda13-dra-2 labels: name: cuda13-dra-2 annotations: gke-gcsfuse/volumes: "true" gke-gcsfuse/cpu-limit: "0" gke-gcsfuse/memory-limit: "0" gke-gcsfuse/ephemeral-storage-limit: "0" spec: tolerations: - key: nvidia.com/gpu operator: Exists effect: NoSchedule - key: kubernetes.io/arch operator: Exists effect: NoSchedule affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: cloud.google.com/gke-accelerator operator: In values: - nvidia-gb200 - key: kubernetes.io/arch operator: In values: - arm64 - key: cloud.google.com/gke-nodepool operator: In values: - a4x-gpu-pool-0 podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: name operator: In values: - gpu-1 - gpu-2 - cuda13-1 - cuda13-2 - cuda13-dra-1 topologyKey: kubernetes.io/hostname hostNetwork: false hostPID: false serviceAccount: workload-identity-k8s-sa volumes: - name: local-ssd hostPath: path: /mnt/stateful_partition/kube-ephemeral-ssd - name: library-dir-host hostPath: path: /home/kubernetes/bin/nvidia - name: gib hostPath: path: /home/kubernetes/bin/gib - name: shared-memory emptyDir: medium: "Memory" sizeLimit: 250Gi - name: data-volume persistentVolumeClaim: claimName: gcs-pvc containers: - image: nvcr.io/nvidia/pytorch:26.04-py3 name: pytorch resources: requests: cpu: 150m limits: nvidia.com/gpu: 4 claims: - name: compute-domain-channel - name: rdma-nics securityContext: privileged: true volumeMounts: - name: local-ssd mountPath: /scratch-data - name: library-dir-host mountPath: /usr/local/nvidia - name: gib mountPath: /usr/local/gib - name: shared-memory mountPath: /dev/shm - name: data-volume mountPath: /data env: - name: N_NODES value: "2" command: ["/bin/bash", "-c"] args: - | export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH} /sbin/ldconfig apt update -y apt install -y iputils-ping openssh-server iproute2 pciutils curl for i in $(seq 0 3); do nvidia-smi -i $i -pm ENABLED done mkdir -p /root/.ssh/ cp /data/a4x-keys/id_rsa /data/a4x-keys/id_rsa.pub /data/a4x-keys/authorized_keys /root/.ssh/ chmod 400 /root/.ssh/id_rsa chmod 600 /root/.ssh/authorized_keys sed -i 's/^#PermitRootLogin prohibit-password/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config service ssh restart echo "export LD_LIBRARY_PATH=/usr/local/gib/lib64:/usr/local/nvidia/lib64:\${LD_LIBRARY_PATH}" | tee -a /root/.bashrc > /dev/null export POSTFIX=$(hostname | cut -d . -f 2-) export WORKERS_BASENAME=$(hostname | cut -d . -f 1 | rev | cut -d - -f 2- | rev) for i in $(seq 1 ${N_NODES}); do OTHER=${WORKERS_BASENAME}-${i} until ssh -o StrictHostKeyChecking=no $OTHER hostname; do echo "Waiting for ${OTHER}..." sleep 10 done echo "${OTHER} slots=4" | tee -a /tmp/hostfile done sleep infinity initContainers: - name: nccl-plugin-installer image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.1.0 imagePullPolicy: Always command: ["/bin/sh", "-c"] args: - | set -ex /scripts/container_entry.sh install --install-nccl cp -R /var/lib/gib/lib64/. /target/usr/local/gib/lib64 cp -R /var/lib/gib/. /target/usr/local/gib volumeMounts: - mountPath: /target/usr/local/gib name: gib - mountPath: /target/dev/shm name: shared-memory resourceClaims: - name: compute-domain-channel resourceClaimTemplateName: cuda13-dra-compute-domain-channel - name: rdma-nics resourceClaimTemplateName: rdma-nics-cuda13-dra-2