从客户提供的 qcow2 镜像出发,构建可在 GCE A4X (Blackwell GB200/GB300) 以及 H100/CPU 实例上通用运行的自定义 ARM64 系统镜像。本指南覆盖从 Builder VM 创建到 A4X 验证的全部步骤,包含 NVIDIA R580 驱动、GVE 1.4.10、IDPF (GB300)、GPU-conditional 服务配置、以及客户 VM 初始化脚本。
gcloud compute images create --source-uri 要求 tar.gz 内含完整 GPT 磁盘映像 (disk.raw)。客户 qcow2 通常使用单分区布局、无 UEFI ESP、且缺少 GCE 所需的 GRUB/cloud 配置。此外,客户定制内核自带的 GVE 驱动版本可能过旧,无法与 GCP GVNIC 通信。因此我们使用"分区 + rsync + chroot 修复"方式,完全控制分区布局、引导链和驱动版本。
设置以下变量后,后续所有命令可直接复制粘贴执行。
# ======================== GCP 项目 ========================
export PROJECT_ID="your-project-id"
export ZONE="us-central1-f" # Builder VM zone (非 A4X zone)
# ======================== Builder VM ========================
export BUILDER_VM="tlinux-builder"
export BUILDER_BOOT_SIZE="200GB" # 需容纳 qcow2 + raw 转换空间
export BUILDER_MACHINE="t2a-standard-8" # ARM64, 8 vCPU 足够编译
# ======================== 目标磁盘 ========================
export TARGET_DISK="tlinux-root-disk"
export TARGET_DISK_SIZE="50GB" # 镜像根盘大小
export DISK_DEVICE="/dev/nvme0n2" # 目标盘在 Builder VM 中的设备路径
export MOUNT_DIR="/mnt/target" # 目标盘挂载点
# ======================== 客户 qcow2 ========================
export QCOW2_FILE="TencentOS-Server-4.6-6.6.119-49.101.nvidia-20260604-Tencent-aarch64.qcow2"
export QCOW2_GCS="" # GCS 路径 (如 gs://bucket/path.qcow2), 留空则用 SCP 上传
# ======================== 镜像命名 ========================
export IMAGE_NAME="tlinux-server-4-gb200-v5"
export SNAPSHOT_NAME="tlinux-snap-v5"
export IMAGE_FAMILY="tlinux-server-4-gb200" # 可选: 镜像系列
# ======================== NVIDIA 驱动 ========================
export NVIDIA_VERSION="580.159.04"
export NVIDIA_CUDA_REPO="https://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa"
# ======================== A4X 验证 (可选) ========================
export A4X_ZONE="us-east1-d"
export RESERVATION="your-reservation-name"
export PLACEMENT_POLICY="your-nvl72-policy"
export GVNIC_NET="your-gvnic-net"
export GVNIC_SUB="your-gvnic-sub"
export GVNIC_NET_1="your-gvnic-net-1"
export GVNIC_SUB_1="your-gvnic-sub-1"
export RDMA_NET="your-rdma-net"
构建过程在一个普通 ARM64 VM 上完成(不需要 GPU)。Builder VM 负责 qcow2 转换、chroot 编译驱动、制作快照。
# 使用 Rocky Linux 9 ARM64 作为 Builder VM 宿主机
# Rocky 9 与 TencentOS 4 共享 EL9 生态, boost-regex/libicu 等依赖可直接获取
gcloud compute instances create "$BUILDER_VM" \
--project="$PROJECT_ID" --zone="$ZONE" \
--machine-type=$BUILDER_MACHINE \
--image-family=rocky-linux-9-optimized-gcp-arm64 \
--image-project=rocky-linux-cloud \
--boot-disk-size=$BUILDER_BOOT_SIZE \
--boot-disk-type=pd-ssd \
--scopes=cloud-platform
# 创建 50GB 目标数据磁盘(最终封装为 GCE 镜像)
gcloud compute disks create "$TARGET_DISK" \
--project="$PROJECT_ID" --zone="$ZONE" \
--size=$TARGET_DISK_SIZE --type=pd-ssd
# 挂载目标磁盘到 Builder VM
gcloud compute instances attach-disk "$BUILDER_VM" \
--disk="$TARGET_DISK" \
--zone="$ZONE" --project="$PROJECT_ID"
lsblk 确认目标盘的设备路径。通常 boot 盘是 /dev/nvme0n1, 附加盘是 /dev/nvme0n2。确认后更新 $DISK_DEVICE 变量。
# SSH 到 Builder VM
gcloud compute ssh "$BUILDER_VM" --zone="$ZONE" --project="$PROJECT_ID" --tunnel-through-iap
# ======== 在 Builder VM 内执行 ========
# 安装必要工具
sudo dnf install -y qemu-img rsync parted gdisk xfsprogs wget curl git podman
# 方式 A:从 GCS 下载 qcow2
# gsutil cp gs://your-bucket/path/$QCOW2_FILE /tmp/
# 方式 B:从本地通过 SCP 上传(在本地机器上执行)
# gcloud compute scp /path/to/$QCOW2_FILE "$BUILDER_VM":/tmp/ \
# --zone="$ZONE" --project="$PROJECT_ID" --tunnel-through-iap
# 验证文件完整性
ls -lh /tmp/$QCOW2_FILE
qemu-img info /tmp/$QCOW2_FILE
file format: qcow2virtual size: 50 GiBdisk size: 5.2 GiB (实际占用空间远小于虚拟大小)
qcow2 是 QEMU 原生格式,无法直接 loop 挂载。需先转换为 raw 格式。
# 转换 qcow2 → raw (耗时取决于虚拟盘大小, 50GB 约需 2-5 分钟)
sudo qemu-img convert -f qcow2 -O raw /tmp/$QCOW2_FILE /tmp/tlinux.raw
echo "Raw file size: $(ls -lh /tmp/tlinux.raw | awk '{print $5}')"
virtual size(非 disk size)。确保 Builder VM boot 盘有足够空间容纳 qcow2 + raw 两个文件。使用 df -h /tmp 检查。
# 创建 loop 设备 (-P 使内核自动扫描分区)
LOOP_DEV=$(sudo losetup --show -fP /tmp/tlinux.raw)
echo "Loop device: $LOOP_DEV"
# 查看分区布局
lsblk "$LOOP_DEV"
sudo fdisk -l "$LOOP_DEV"
根据分区布局挂载根分区:
# 典型情况 1: 单分区 qcow2 (仅 p1 为 / 根分区)
sudo mkdir -p /mnt/qcow2
sudo mount "${LOOP_DEV}p1" /mnt/qcow2
# 典型情况 2: 含 EFI 分区的 qcow2 (p1=EFI, p2=/)
# sudo mount "${LOOP_DEV}p2" /mnt/qcow2
# 验证挂载成功
ls /mnt/qcow2/etc/os-release
cat /mnt/qcow2/etc/os-release | head -5
# 检查关键文件是否存在
echo "=== 内核版本 ==="
ls /mnt/qcow2/lib/modules/
echo "=== kernel-devel ==="
ls /mnt/qcow2/usr/src/kernels/ 2>/dev/null || echo "未预装 kernel-devel (后续需安装)"
echo "=== 是否已含 NVIDIA 驱动 ==="
find /mnt/qcow2/lib/modules -name "nvidia.ko*" 2>/dev/null || echo "未包含 (需 Step 6 编译)"
echo "=== SSH 端口 ==="
grep -E "^Port " /mnt/qcow2/etc/ssh/sshd_config 2>/dev/null || echo "使用默认 22"
/lib/modules/ 下的版本号,后续驱动编译依赖此值在 50GB 目标磁盘上创建 UEFI 兼容的 GPT 分区表。
# 1. 创建 GPT 分区: ESP (512MB, FAT32) + 根分区 (剩余, XFS)
sudo parted -s "$DISK_DEVICE" mklabel gpt
sudo parted -s "$DISK_DEVICE" mkpart primary fat32 1MiB 513MiB
sudo parted -s "$DISK_DEVICE" set 1 esp on
sudo parted -s "$DISK_DEVICE" mkpart primary xfs 513MiB 100%
sudo partprobe "$DISK_DEVICE"
sleep 3
# 2. 格式化
sudo mkfs.vfat -F32 "${DISK_DEVICE}p1"
sudo mkfs.xfs -f "${DISK_DEVICE}p2"
# 3. 挂载
sudo mkdir -p "$MOUNT_DIR"
sudo mount "${DISK_DEVICE}p2" "$MOUNT_DIR"
sudo mkdir -p "$MOUNT_DIR/boot/efi"
sudo mount "${DISK_DEVICE}p1" "$MOUNT_DIR/boot/efi"
echo "Target disk mounted at $MOUNT_DIR"
df -h "$MOUNT_DIR" "$MOUNT_DIR/boot/efi"
# 将 qcow2 根文件系统完整复制到目标磁盘
# -a: 归档模式 (保留权限/所有者/符号链接)
# -H: 保留硬链接
# -A: 保留 ACL
# -X: 保留 xattr
# -x: 不跨文件系统边界
sudo rsync -aHAXx --info=progress2 /mnt/qcow2/ "$MOUNT_DIR/"
# 卸载 qcow2 loop 挂载
sudo umount /mnt/qcow2
sudo losetup -d "$LOOP_DEV"
# 可选: 删除 raw 文件节省空间
sudo rm -f /tmp/tlinux.raw
echo "rsync complete. Target disk usage:"
du -sh "$MOUNT_DIR"
进入 chroot 环境修复引导配置、禁用 cloud-init、配置 GCE 兼容网络。
# 绑定挂载宿主机虚拟 API
sudo mount --bind /dev "$MOUNT_DIR/dev"
sudo mount --bind /dev/pts "$MOUNT_DIR/dev/pts"
sudo mount --bind /proc "$MOUNT_DIR/proc"
sudo mount --bind /sys "$MOUNT_DIR/sys"
sudo mount --bind /sys/firmware/efi/efivars "$MOUNT_DIR/sys/firmware/efi/efivars" 2>/dev/null || true
# 同步 DNS 解析 (chroot 内需要网络访问)
sudo cp /etc/resolv.conf "$MOUNT_DIR/etc/resolv.conf"
# 禁用 TencentOS 软件源 (mirrors.tencentyun.com / mirrors.tencent.com 从 GCE 无法访问)
for repo in "$MOUNT_DIR/etc/yum.repos.d/TencentOS"*.repo "$MOUNT_DIR/etc/yum.repos.d/epol"*.repo; do
[ -f "$repo" ] && sudo sed -i 's/enabled=1/enabled=0/g' "$repo"
done
# 同时禁用任何指向 tencent 镜像的 repo
grep -rl 'tencentyun.com\|mirrors.tencent.com' "$MOUNT_DIR/etc/yum.repos.d/" 2>/dev/null | \
xargs -I{} sudo sed -i 's/enabled=1/enabled=0/g' {}
重写 fstab 和 GRUB 配置,使用目标磁盘的 UUID。
# 从 host 侧获取目标磁盘分区 UUID 并写入 chroot
ROOT_UUID=$(blkid -o value -s UUID "${DISK_DEVICE}p2")
EFI_UUID=$(blkid -o value -s UUID "${DISK_DEVICE}p1")
echo "Root UUID: $ROOT_UUID"
echo "EFI UUID: $EFI_UUID"
cat <
GRUB_ENABLE_BLSCFG=false 是必须的。qcow2 rsync 过来的系统可能没有正确的 BLS 入口文件 (/boot/loader/entries/*.conf)。禁用 BLS 后,grub2-mkconfig 直接扫描 /boot/vmlinuz-* 生成 menuentry,更可靠。
# 检测内核版本 (从 host 侧读取, 不依赖 chroot 内 rpm)
KERNEL_VER=$(ls "$MOUNT_DIR/lib/modules/" | head -1)
echo "Detected kernel: $KERNEL_VER"
# 确保 vmlinuz 在 /boot/ 下, 生成 initramfs, 构建 GRUB 配置
# 注意: 使用不带引号的 heredoc, 使 $KERNEL_VER 在 host 侧展开
sudo chroot "$MOUNT_DIR" /bin/bash <
<<CHROOT_BOOT(不带引号)= host 侧展开 $KERNEL_VER。<<'CHROOT_BOOT'(带引号)= 变量不展开,在 chroot 内执行时引用。客户 qcow2 配置了腾讯云 cloud-init (metadata: 169.254.0.23), 会干扰 GCE NetworkManager 和 sshd 密钥生成。必须彻底禁用。
sudo chroot "$MOUNT_DIR" /bin/bash <<'EOF'
set -ex
# ===== 1. 彻底禁用 cloud-init =====
touch /etc/cloud/cloud-init.disabled
mkdir -p /etc/cloud/cloud.cfg.d
cat > /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg <<'CLOUDINIT'
network: {config: disabled}
CLOUDINIT
for svc in cloud-init.service cloud-init-local.service cloud-config.service cloud-final.service cloud-init.target; do
systemctl mask "$svc" 2>/dev/null || true
done
# ===== 2. SSH 端口修复 =====
# 客户 qcow2 可能使用非标端口 (36000/56000), 改回 22
sed -i 's/^Port .*/Port 22/' /etc/ssh/sshd_config
grep -q "^Port " /etc/ssh/sshd_config || echo "Port 22" >> /etc/ssh/sshd_config
# Mask 客户自定义端口 sshd 服务
systemctl mask sshd-56000.service 2>/dev/null || true
systemctl mask sshd-36000.service 2>/dev/null || true
# 修复 sshd-keygen 与 cloud-init 的条件依赖
# cloud-init.target 被 mask 后, sshd-keygen 的 ConditionPathExists 检查可能误判
for algo in rsa ecdsa ed25519; do
mkdir -p "/etc/systemd/system/sshd-keygen@${algo}.service.d"
cat > "/etc/systemd/system/sshd-keygen@${algo}.service.d/override.conf" <<'OVERRIDE'
[Unit]
ConditionPathExists=
OVERRIDE
done
# ===== 3. NetworkManager GVNIC 配置 =====
# GCE A4X 第一块 GVNIC 接口名为 enp0s3 (不是 eth0)
cat > /etc/NetworkManager/system-connections/enp0s3-gvnic.nmconnection <<'NMCONN'
[connection]
id=enp0s3-gvnic
type=ethernet
interface-name=enp0s3
autoconnect=true
autoconnect-priority=100
[ipv4]
method=auto
[ipv6]
method=auto
NMCONN
chmod 600 /etc/NetworkManager/system-connections/enp0s3-gvnic.nmconnection
# 全局自动连接
cat > /etc/NetworkManager/conf.d/99-dhcp-all.conf <<'NMDHCP'
[main]
no-auto-default=
autoconnect-retries-default=2
[device]
match-device=type:ethernet
managed=1
NMDHCP
# ===== 4. SELinux 与防火墙 =====
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config 2>/dev/null || true
systemctl disable firewalld 2>/dev/null || true
systemctl mask firewalld 2>/dev/null || true
# ===== 5. memlock unlimited (RDMA 必须) =====
cat > /etc/security/limits.d/rdma.conf <<'LIMITS'
* soft memlock unlimited
* hard memlock unlimited
LIMITS
echo "Cloud-init disabled, network fixed, SSH on port 22"
EOF
Google 预编译的 Guest Agent 和 OS Login NSS 依赖特定版本的共享库 (boost-regex 1.75, libicu 67), 而 TencentOS 4.0 系统提供更高版本 (boost 1.82, ICU 71/72)。直接安装会导致动态链接崩溃。必须从 Rocky 9 宿主机提取正确版本并注入。
ln -s libicu.so.71 libicu.so.67 会因 ABI 不兼容导致 google_oslogin_nss_cache 运行时崩溃 (exit 127), 阻断 SSH via IAP 登录。必须注入原版 libicu 67 共享库。
# 1. 从 Rocky 9 宿主机获取 boost-regex (dnf download 比硬编码 URL 更可靠)
# 注意: TencentOS 自带 boost-regex 1.82, 但 GCE Agent 需要 1.75
# 必须使用 --force 强制安装旧版本 (两个版本 .so 文件共存)
dnf download boost-regex --destdir=/tmp 2>/dev/null || true
BOOST_RPM=$(ls /tmp/boost-regex-*.aarch64.rpm 2>/dev/null | head -1)
if [ -n "$BOOST_RPM" ]; then
sudo cp "$BOOST_RPM" "$MOUNT_DIR/tmp/"
sudo chroot "$MOUNT_DIR" rpm -ivh --force --nodeps /tmp/$(basename "$BOOST_RPM")
sudo rm -f "$MOUNT_DIR/tmp/$(basename "$BOOST_RPM")"
echo "boost-regex 1.75 injected"
else
echo "WARNING: boost-regex download failed"
fi
# 2. 从 Rocky 9 宿主机获取 libicu 67 并解包注入
dnf download libicu --destdir=/tmp 2>/dev/null || true
ICU_RPM=$(ls /tmp/libicu-67.1-*.aarch64.rpm 2>/dev/null | head -1)
if [ -n "$ICU_RPM" ]; then
mkdir -p /tmp/icu67-extract
rpm2cpio "$ICU_RPM" | cpio -idmv -D /tmp/icu67-extract/
sudo cp -d /tmp/icu67-extract/usr/lib64/libicu*.so.67* "$MOUNT_DIR/usr/lib64/"
rm -rf /tmp/icu67-extract "$ICU_RPM"
echo "libicu 67 injected"
else
echo "WARNING: libicu download failed"
fi
# 3. 从 Rocky 9 宿主机注入 jq (google-compute-engine 依赖, TencentOS 源已禁用)
for pkg in jq oniguruma; do
dnf download "$pkg" --destdir=/tmp 2>/dev/null || true
done
for rpm_file in /tmp/jq-*.aarch64.rpm /tmp/oniguruma-*.aarch64.rpm; do
[ -f "$rpm_file" ] || continue
sudo cp "$rpm_file" "$MOUNT_DIR/tmp/"
sudo chroot "$MOUNT_DIR" rpm -ivh --force --nodeps /tmp/$(basename "$rpm_file")
sudo rm -f "$MOUNT_DIR/tmp/$(basename "$rpm_file")"
done
# 4. 安装 GCE Guest Agent
sudo chroot "$MOUNT_DIR" /bin/bash <<'EOF'
set -ex
# 验证 OS Login 依赖已满足
ldd /usr/bin/google_oslogin_nss_cache 2>/dev/null || echo "oslogin binary will be installed next"
# 配置 Google 软件源
cat > /etc/yum.repos.d/google-cloud.repo <<'GCREPO'
[google-compute-engine]
name=Google Compute Engine
baseurl=https://packages.cloud.google.com/yum/repos/google-compute-engine-el9-aarch64-stable
enabled=1
gpgcheck=0
[google-cloud-sdk]
name=Google Cloud SDK
baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el9-aarch64
enabled=1
gpgcheck=0
GCREPO
# 安装 GCE 核心组件
dnf install -y google-compute-engine google-guest-agent google-osconfig-agent
# 启用自启动服务
systemctl enable NetworkManager sshd
systemctl enable google-guest-agent google-startup-scripts google-shutdown-scripts google-osconfig-agent
# 最终验证: OS Login NSS 缓存所有动态库已就绪
echo "=== ldd google_oslogin_nss_cache ==="
ldd /usr/bin/google_oslogin_nss_cache
EOF
Blackwell GB200/GB300 高速互联架构要求 Open 类型驱动 (--kernel-module-type=open)。
# 1. Host 侧下载驱动安装器 (~314MB)
wget -O "$MOUNT_DIR/tmp/nvidia.run" \
"https://us.download.nvidia.com/tesla/${NVIDIA_VERSION}/NVIDIA-Linux-aarch64-${NVIDIA_VERSION}.run"
# 2. 确保内核版本变量已设置
KERNEL_VER=$(ls "$MOUNT_DIR/lib/modules/" | head -1)
echo "Building NVIDIA $NVIDIA_VERSION against kernel: $KERNEL_VER"
# 3. Chroot 内编译 (使用不带引号 heredoc, $KERNEL_VER 在 host 侧展开)
sudo chroot "$MOUNT_DIR" /bin/bash < /etc/modprobe.d/blacklist-nouveau.conf <<'NOUVEAU'
blacklist nouveau
options nouveau modeset=0
NOUVEAU
# 确保 kernel-devel 已安装
if [ ! -d "/lib/modules/${KERNEL_VER}/build" ]; then
echo "kernel-devel missing, installing..."
dnf install -y kernel-devel-${KERNEL_VER} kernel-headers-${KERNEL_VER}
fi
# 编译安装 Open 驱动
sh /tmp/nvidia.run \
--silent \
--no-questions \
--accept-license \
--kernel-module-type=open \
--kernel-name="${KERNEL_VER}" \
--kernel-source-path="/lib/modules/${KERNEL_VER}/build" \
--utility-prefix=/usr \
--opengl-prefix=/usr
rm -f /tmp/nvidia.run
# 验证驱动类型
echo "=== Verify OPEN type ==="
for mod in nvidia nvidia-modeset nvidia-uvm nvidia-drm nvidia-peermem; do
modfile=\$(find /lib/modules/${KERNEL_VER} -name "\${mod}.ko*" 2>/dev/null | head -1)
if [ -n "\$modfile" ]; then
license=\$(modinfo -F license "\$modfile" 2>/dev/null || echo "unknown")
echo " \${mod}: \${license}"
fi
done
echo "=== Verify nvidia-smi binary ==="
ls -la /usr/bin/nvidia-smi
CHROOT_NVIDIA
GCE A4X 使用 Google Virtual Ethernet (GVE/GVNIC)。客户定制内核自带的 GVE 版本通常过旧 (如 v1.0.0), 无法与 GCP GVNIC 硬件通信 (串口日志: AQ commands timed out, Could not verify driver compatibility)。必须从 Google 官方仓库编译 v1.4.10。
netdev_queue_mgmt_ops, netmem_tx 等)。对于 6.6 内核,需用 Coccinelle 补丁实现向后兼容。Rocky 9 aarch64 无法安装 coccinelle, 解决方案: podman + Fedora 容器。
# 在 Host 侧 (非 chroot) 执行
# 1. 克隆 Google GVE 源码
cd /tmp
git clone https://github.com/GoogleCloudPlatform/compute-virtual-ethernet-linux.git
# 2. 在 Fedora 容器中运行 coccinelle 补丁 + 生成兼容源码
podman run --rm -v /tmp/compute-virtual-ethernet-linux:/src:Z fedora:41 bash -c '
dnf install -y coccinelle make
cd /src
bash build_src.sh --target=oot --version=1.4.10
'
# 3. 复制到 chroot 内
sudo cp -r /tmp/compute-virtual-ethernet-linux/build "$MOUNT_DIR/tmp/gve-build"
# 4. Chroot 内编译
KERNEL_VER=$(ls "$MOUNT_DIR/lib/modules/" | head -1)
sudo chroot "$MOUNT_DIR" bash -c "
cd /tmp/gve-build
make BUILD_KERNEL=$KERNEL_VER
"
# 5. 替换旧 GVE 模块
GVE_DIR="$MOUNT_DIR/lib/modules/$KERNEL_VER/kernel/drivers/net/ethernet/google/gve"
sudo mkdir -p "$GVE_DIR"
sudo mv "$GVE_DIR/gve.ko.xz" "$GVE_DIR/gve.ko.xz.bak" 2>/dev/null || true
sudo mv "$GVE_DIR/gve.ko" "$GVE_DIR/gve.ko.bak" 2>/dev/null || true
sudo cp "$MOUNT_DIR/tmp/gve-build/gve.ko" "$GVE_DIR/gve.ko"
# 6. 更新模块依赖并重建 initramfs (含新 GVE)
sudo chroot "$MOUNT_DIR" bash -c "
depmod -a $KERNEL_VER
dracut --force --kver $KERNEL_VER
"
# 7. 清理
sudo rm -rf "$MOUNT_DIR/tmp/gve-build"
rm -rf /tmp/compute-virtual-ethernet-linux
echo "GVE 1.4.10-oot compiled and installed"
GVE version 1.4.10-oot 和 Driver is running with DQO RDA queue format。modinfo gve 显示 version: 1.4.10-oot。
Blackwell GPU 跨卡 NVLink 通信需要 IMEX 通道进程。GCP A4X 的 NVSwitch 由 hypervisor Service VM 管理, Guest VM 无需运行 Fabric Manager。
# 1. 下载 nvidia-modprobe (IMEX 的 RPM 依赖)
wget -O "$MOUNT_DIR/tmp/nvidia-modprobe.rpm" \
"${NVIDIA_CUDA_REPO}/nvidia-modprobe-${NVIDIA_VERSION}-1.el9.aarch64.rpm"
sudo chroot "$MOUNT_DIR" rpm -ivh --nodeps /tmp/nvidia-modprobe.rpm
# 2. 下载并安装 IMEX
wget -O "$MOUNT_DIR/tmp/nvidia-imex.rpm" \
"${NVIDIA_CUDA_REPO}/nvidia-imex-${NVIDIA_VERSION}-1.el9.aarch64.rpm"
sudo chroot "$MOUNT_DIR" /bin/bash <<'EOF'
set -ex
dnf localinstall -y /tmp/nvidia-imex.rpm
# IMEX 通道标志 — Blackwell 必须
echo "options nvidia NVreg_CreateImexChannel0=1" > /etc/modprobe.d/nvidia-imex.conf
# 启用 IMEX 服务
systemctl enable nvidia-imex
rm -f /tmp/nvidia-imex.rpm /tmp/nvidia-modprobe.rpm
echo "IMEX installed"
EOF
GDRCopy 提供 GPU 显存到 CPU 内存的低延迟拷贝, 是部分 NCCL/UCX 操作的可选加速路径。
KERNEL_VER=$(ls "$MOUNT_DIR/lib/modules/" | head -1)
# 1. 安装编译依赖 (在 chroot 内)
sudo chroot "$MOUNT_DIR" /bin/bash <&1 | tail -5
CHROOT_GDRCOPY_DEPS
# 2. 克隆并编译 GDRCopy (从 host 侧操作更可靠)
GDRCOPY_DIR="$MOUNT_DIR/tmp/gdrcopy"
if [ ! -d "$GDRCOPY_DIR" ]; then
git clone --branch v2.6 --depth 1 https://github.com/NVIDIA/gdrcopy.git "$GDRCOPY_DIR"
fi
sudo chroot "$MOUNT_DIR" /bin/bash <&1 | tail -5
make CUDA="\$CUDA_PATH" KVER=${KERNEL_VER} prefix=/usr/local/gdrcopy driver 2>&1 | tail -5
make CUDA="\$CUDA_PATH" KVER=${KERNEL_VER} drv_install 2>&1 | tail -5
# 编译安装 userspace tools (copylat, copybw, sanity 等)
# 需要 nvcc: dnf install -y cuda-nvcc-13-0
make CUDA="\$CUDA_PATH" prefix=/usr/local/gdrcopy exes 2>&1 | tail -5
make CUDA="\$CUDA_PATH" prefix=/usr/local/gdrcopy exes_install 2>&1 | tail -5
# 配置自动加载
echo "gdrdrv" > /etc/modules-load.d/gdrdrv.conf
# 安装 systemd service
mkdir -p /usr/libexec/gdrcopy
cp /tmp/gdrcopy/packages/rhel/init.d/gdrcopy /usr/libexec/gdrcopy/gdrcopy
chmod +x /usr/libexec/gdrcopy/gdrcopy
cp /tmp/gdrcopy/packages/rhel/gdrcopy.service /etc/systemd/system/gdrcopy.service
systemctl daemon-reload
systemctl enable gdrcopy
# ldconfig
echo "/usr/local/gdrcopy/lib" > /etc/ld.so.conf.d/gdrcopy.conf
ldconfig
echo "GDRCopy v2.6 installed"
CHROOT_GDRCOPY
# 清理源码
sudo rm -rf "$GDRCOPY_DIR"
GB300 实例使用 Intel IDPF (Infrastructure Data-Plane Function) NIC。TLinux 内核 6.6.x 不包含 IDPF 驱动(该驱动在 Linux 6.7 才合入主线),需从 Intel 官方仓库编译 out-of-tree 模块。
KERNEL_VER=$(ls "$MOUNT_DIR/lib/modules/" | head -1)
# 1. 下载 Intel IDPF v1.0.11 源码
cd /tmp
curl -sL -o idpf-v1.0.11.tar.gz \
https://github.com/intel/ethernet-linux-idpf/archive/refs/tags/v1.0.11.tar.gz
tar xzf idpf-v1.0.11.tar.gz
# 2. 复制源码到 chroot
sudo cp -r /tmp/ethernet-linux-idpf-1.0.11 "$MOUNT_DIR/tmp/idpf-src"
# 3. Chroot 内编译
sudo chroot "$MOUNT_DIR" /bin/bash <&1 | tail -3
# 编译 IDPF 模块 (BUILD_KERNEL 必须显式指定, 否则 Makefile 用 uname -r 取宿主机内核)
make -j\$(nproc) BUILD_KERNEL=${KERNEL_VER} 2>&1 | tail -10
# 安装到内核模块目录
make install BUILD_KERNEL=${KERNEL_VER} 2>&1 | tail -5
# 验证
modinfo -k ${KERNEL_VER} idpf | head -5
echo "IDPF v1.0.11 compiled and installed for kernel ${KERNEL_VER}"
CHROOT_IDPF
# 4. 清理
sudo rm -rf "$MOUNT_DIR/tmp/idpf-src"
rm -rf /tmp/ethernet-linux-idpf-1.0.11 /tmp/idpf-v1.0.11.tar.gz
modinfo idpf 应显示 version: 1.0.11、description: Intel(R) Infrastructure Data Path Function Linux Driver。/lib/modules/<KERNEL_VER>/updates/drivers/net/ethernet/intel/idpf/src/idpf.ko
安装 Lustre 文件系统客户端,用于挂载 GCP Cloud Lustre (Managed Lustre)。使用 DKMS 方式编译内核模块,以兼容 TLinux 自定义内核。
kmod-lustre-client RPM — 它是为 Rocky 9 标准内核编译的,与 TLinux 内核符号不兼容。必须使用 lustre-client-dkms (DKMS 方式) 在安装时编译。
KERNEL_VER=$(ls "$MOUNT_DIR/lib/modules/" | head -1)
# 1. 下载 dkms (TLinux/Rocky 默认 repo 不包含, 从 EPEL 获取)
cd /tmp
curl -sLO https://dl.fedoraproject.org/pub/epel/9/Everything/aarch64/Packages/d/dkms-3.4.1-1.el9.noarch.rpm
sudo cp dkms-3.4.1-1.el9.noarch.rpm "$MOUNT_DIR/tmp/"
# 2. 下载编译依赖 (从 Rocky 9 host 下载, 注入 chroot)
sudo dnf config-manager --set-enabled crb 2>/dev/null || true
sudo dnf download --destdir=/tmp/lustre-deps --arch=aarch64,noarch \
json-c-devel libmount-devel libyaml-devel 2>&1 | tail -5
sudo mkdir -p "$MOUNT_DIR/tmp/lustre-deps"
sudo cp /tmp/lustre-deps/*.rpm "$MOUNT_DIR/tmp/lustre-deps/"
# 3. Chroot 内安装
sudo chroot "$MOUNT_DIR" /bin/bash < /etc/yum.repos.d/lustre-client.repo << 'EOF'
[lustre-client]
name=GCP Lustre Client
baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-9
enabled=1
gpgcheck=0
repo_gpgcheck=0
EOF
# 安装 lustre-client-dkms (auto-build 会因 host kernel 不匹配而失败, 正常)
dnf install -y lustre-client-dkms-2.14.0_ddn256-1.el9 || true
# 手动为正确的 kernel 构建
dkms build lustre-client/2.14.0_ddn256 -k ${KERNEL_VER} 2>&1 | tail -5
dkms install lustre-client/2.14.0_ddn256 -k ${KERNEL_VER} 2>&1 | tail -5
# 安装 lustre-client userspace (排除不兼容的 kmod)
dnf install -y --exclude='kmod-lustre-client*' lustre-client-2.14.0_ddn256-1.el9
# 验证
dkms status -k ${KERNEL_VER} | grep lustre
modinfo -k ${KERNEL_VER} lustre | head -3
echo "Lustre Client DDN 2.14 installed"
CHROOT_LUSTRE
# 4. 清理
sudo rm -rf "$MOUNT_DIR/tmp/lustre-deps" "$MOUNT_DIR/tmp/dkms-*.rpm"
rm -rf /tmp/lustre-deps /tmp/dkms-*.rpm
dkms status -k $KERNEL_VER | grep lustre 应显示 lustre-client/2.14.0_ddn256 ... installedmodinfo -k $KERNEL_VER lustre 应显示 version: 2.14.0_ddn256rpm -qa | grep lustre 应有 lustre-client 和 lustre-client-dkms
modprobe lnet && modprobe lustre && lnetctl lnet configure && mount -t lustre <IP>@tcp:/lustrefs /mnt/lustre
镜像中 bake 了 gdrcopy.service 和 nvidia-imex.service,它们在 GPU 机型上正常工作,但在非 GPU 实例(H100/CPU 或 GB300 首次配置时)会因缺少 nvidia 内核模块而报错。通过添加 systemd ConditionPathExists 条件,使这些服务仅在 GPU 可用时启动。
sudo chroot "$MOUNT_DIR" /bin/bash <<'EOF'
set -ex
# gdrcopy.service: 仅在 nvidia 驱动已加载时启动
mkdir -p /etc/systemd/system/gdrcopy.service.d
cat > /etc/systemd/system/gdrcopy.service.d/gpu-only.conf <<'DROPIN'
[Unit]
ConditionPathExists=/proc/driver/nvidia
DROPIN
# nvidia-imex.service: 同样仅在 GPU 可用时启动
mkdir -p /etc/systemd/system/nvidia-imex.service.d
cat > /etc/systemd/system/nvidia-imex.service.d/gpu-only.conf <<'DROPIN'
[Unit]
ConditionPathExists=/proc/driver/nvidia
DROPIN
systemctl daemon-reload
echo "GPU-conditional drop-ins installed"
echo " - gdrcopy: ConditionPathExists=/proc/driver/nvidia"
echo " - nvidia-imex: ConditionPathExists=/proc/driver/nvidia"
EOF
/proc/driver/nvidia 仅在 nvidia 内核模块加载后存在。GPU 实例上 nvidia 驱动自动加载后该路径出现,服务正常启动。非 GPU 实例上该路径不存在,systemd 跳过服务启动(Condition: start condition unmet),不会产生 failed 状态。
删除 chroot 残留的临时文件, 按依赖顺序卸载所有挂载点。
# 1. 清除临时文件
sudo rm -f "$MOUNT_DIR/etc/resolv.conf"
sudo rm -f "$MOUNT_DIR/etc/yum.repos.d/tlinux4.repo"
# 2. 按依赖顺序卸载 (先子后父)
sudo umount "$MOUNT_DIR/sys/firmware/efi/efivars" 2>/dev/null || true
sudo umount "$MOUNT_DIR/sys" 2>/dev/null || true
sudo umount "$MOUNT_DIR/proc" 2>/dev/null || true
sudo umount "$MOUNT_DIR/dev/pts" 2>/dev/null || true
sudo umount "$MOUNT_DIR/dev" 2>/dev/null || true
sudo umount "$MOUNT_DIR/boot/efi" 2>/dev/null || true
sudo umount "$MOUNT_DIR" 2>/dev/null || true
# 3. 验证所有挂载已清理
mount | grep "$MOUNT_DIR" || echo "Success: 磁盘已完全安全卸载"
将目标磁盘快照并封装为 GCE 自定义镜像。
# 1. 创建磁盘快照
gcloud compute snapshots create "$SNAPSHOT_NAME" \
--project="$PROJECT_ID" \
--source-disk="$TARGET_DISK" \
--source-disk-zone="$ZONE" \
--storage-location="us"
# 2. 从快照创建镜像 (声明 UEFI + GVNIC + IDPF 特性)
gcloud compute images create "$IMAGE_NAME" \
--project="$PROJECT_ID" \
--source-snapshot="$SNAPSHOT_NAME" \
--guest-os-features="UEFI_COMPATIBLE,GVNIC,IDPF" \
--architecture="ARM64" \
--storage-location="us" \
--family="$IMAGE_FAMILY"
# 验证
gcloud compute images describe "$IMAGE_NAME" --project="$PROJECT_ID" \
--format="table(name,status,archiveSizeBytes,guestOsFeatures[].type)"
--family 将镜像加入系列。后续创建 VM 时可用 --image-family 自动选择最新版本, 方便迭代。
v5.3 在 v5.2 基础上新增 nvidia-persistenced systemd 服务,使 startup script 不再需要 IMEX/GDRCopy/fabricmanager 配置和重启。
ConditionPathExists=/proc/driver/nvidia), 替代 startup script 中的 nvidia-smi -pm ENABLED# 使用已有 builder VM (us-central1-f)
BUILDER="tlinux-builder-v5"
BUILDER_ZONE="us-central1-f"
PROJECT_ID="gpu-launchpad-playground"
# 1. 从 v5.2 镜像创建磁盘
gcloud compute disks create tlinux-root-disk-v5dot3 \
--project="$PROJECT_ID" --zone="$BUILDER_ZONE" \
--image=tlinux-server-4-gb200-v5dot3 \
--image-project="$PROJECT_ID" \
--size=1000GB --type=pd-ssd
# 2. 挂载到 builder VM
gcloud compute instances attach-disk "$BUILDER" \
--project="$PROJECT_ID" --zone="$BUILDER_ZONE" \
--disk=tlinux-root-disk-v5dot3 --device-name=v5dot3
# 3. SSH 到 builder VM
gcloud compute ssh "$BUILDER" --zone="$BUILDER_ZONE" \
--tunnel-through-iap --project="$PROJECT_ID"
# 在 builder VM 上执行:
MOUNT_DIR="/mnt/tlinux-root"
sudo mkdir -p "$MOUNT_DIR"
# 挂载 (v5.2 磁盘为 GPT 多分区, root 在 p2)
DEV=$(readlink -f /dev/disk/by-id/google-v5dot3)
sudo mount "${DEV}p2" "$MOUNT_DIR"
sudo mount "${DEV}p1" "$MOUNT_DIR/boot/efi"
sudo mount --bind /dev "$MOUNT_DIR/dev"
sudo mount --bind /dev/pts "$MOUNT_DIR/dev/pts"
sudo mount --bind /proc "$MOUNT_DIR/proc"
sudo mount --bind /sys "$MOUNT_DIR/sys"
# chroot 进入
sudo chroot "$MOUNT_DIR" /bin/bash
# 在 chroot 环境中:
# 添加 nvidia-persistenced 服务 (GPU-conditional)
cat > /etc/systemd/system/nvidia-persistenced.service <<'EOF'
[Unit]
Description=NVIDIA Persistence Daemon
ConditionPathExists=/proc/driver/nvidia
After=nvidia.service
[Service]
Type=forking
ExecStart=/usr/bin/nvidia-persistenced --persistence-mode
ExecStopPost=/bin/rm -rf /var/run/nvidia-persistenced
[Install]
WantedBy=multi-user.target
EOF
systemctl enable nvidia-persistenced
# 退出 chroot
exit
# 清理挂载 (回到 builder VM)
sudo umount "$MOUNT_DIR/sys" 2>/dev/null || true
sudo umount "$MOUNT_DIR/proc" 2>/dev/null || true
sudo umount "$MOUNT_DIR/dev/pts" 2>/dev/null || true
sudo umount "$MOUNT_DIR/dev" 2>/dev/null || true
sudo umount "$MOUNT_DIR/boot/efi" 2>/dev/null || true
sudo umount "$MOUNT_DIR" 2>/dev/null || true
# 卸载磁盘
exit # 退出 SSH
# 创建快照和镜像
gcloud compute instances detach-disk "$BUILDER" \
--project="$PROJECT_ID" --zone="$BUILDER_ZONE" \
--disk=tlinux-root-disk-v5dot3
gcloud compute snapshots create tlinux-v5dot3-snapshot \
--project="$PROJECT_ID" \
--source-disk=tlinux-root-disk-v5dot3 \
--source-disk-zone="$BUILDER_ZONE" \
--storage-location="us"
gcloud compute images create tlinux-server-4-gb200-v5dot3 \
--project="$PROJECT_ID" \
--source-snapshot=tlinux-v5dot3-snapshot \
--guest-os-features="UEFI_COMPATIBLE,GVNIC,IDPF" \
--architecture="ARM64" \
--storage-location="us" \
--family="tlinux-server-4-gb200"
# 验证
gcloud compute images describe tlinux-server-4-gb200-v5dot3 \
--project="$PROJECT_ID" \
--format="table(name,status,archiveSizeBytes,guestOsFeatures[].type)"
| 属性 | 值 |
|---|---|
| 镜像名称 | tlinux-server-4-gb200-v5dot3 |
| 所属项目 | dave-selfstudy01 (已共享至 gpu-launchpad-playground) |
| 镜像系列 (Family) | tlinux-server-4-gb200 |
| 架构 | ARM64 |
| 镜像大小 | 50 GB (创建 VM 时 boot-disk-size 建议 ≥ 1000 GB) |
| Guest OS Features | UEFI_COMPATIBLE, GVNIC, IDPF |
| 内核 | 6.6.119-49.101.nvidia.tl4.aarch64 |
| NVIDIA 驱动 | 580.159.04 (Open) |
| GVE | v1.4.10-oot |
| IDPF | v1.0.11 (Intel out-of-tree, GB300 用) |
| GDRCopy | v2.6 (GPU-conditional) |
| IMEX | enabled (GPU-conditional) |
| nvidia-persistenced | enabled (GPU-conditional, v5.3 新增) |
| Lustre Client | DDN 2.14 (DKMS, 不自动挂载) |
| 兼容机型 | A4X/GB200, GB300, H100, CPU-only |
nvidia-smi -pm ENABLEDgcloud compute instances create YOUR_VM_NAME \
--project=gpu-launchpad-playground \
--image=tlinux-server-4-gb200-v5dot3 \
--image-project=dave-selfstudy01 \
--boot-disk-size=1000GB --boot-disk-type=hyperdisk-balanced \
... # 其他参数
gcloud compute instances create YOUR_VM_NAME \
--project=gpu-launchpad-playground \
--image-family=tlinux-server-4-gb200 \
--image-project=dave-selfstudy01 \
--boot-disk-size=1000GB --boot-disk-type=hyperdisk-balanced \
... # 其他参数
gcloud 会报错: "Requested disk size cannot be smaller than the image size (1000 GB)"。
# 变量 (根据实际环境修改)
PROJECT="gpu-launchpad-playground"
ZONE="us-east1-d" # A4X 可用区
RESERVATION="your-reservation-name"
PLACEMENT_POLICY="your-nvl72-policy"
GVNIC_NET="your-gvnic-net"
GVNIC_SUB="your-gvnic-sub"
GVNIC_NET_1="your-gvnic-net-1"
GVNIC_SUB_1="your-gvnic-sub-1"
RDMA_NET="your-rdma-net"
gcloud compute instances create worker-d1-w0 \
--project=$PROJECT --zone=$ZONE \
--machine-type=a4x-highgpu-4g \
--image=tlinux-server-4-gb200-v5dot3 \
--image-project=dave-selfstudy01 \
--boot-disk-size=1000GB --boot-disk-type=hyperdisk-balanced \
--scopes=cloud-platform \
--reservation-affinity=specific --reservation=$RESERVATION \
--provisioning-model=RESERVATION_BOUND \
--instance-termination-action=STOP \
--maintenance-policy=TERMINATE \
--restart-on-failure \
--resource-policies=$PLACEMENT_POLICY \
--network-interface=nic-type=GVNIC,network=$GVNIC_NET,subnet=$GVNIC_SUB \
--network-interface=nic-type=GVNIC,network=$GVNIC_NET_1,subnet=$GVNIC_SUB_1,no-address \
--network-interface=nic-type=MRDMA,network=$RDMA_NET,subnet=${RDMA_NET}-sub-0,no-address \
--network-interface=nic-type=MRDMA,network=$RDMA_NET,subnet=${RDMA_NET}-sub-1,no-address \
--network-interface=nic-type=MRDMA,network=$RDMA_NET,subnet=${RDMA_NET}-sub-2,no-address \
--network-interface=nic-type=MRDMA,network=$RDMA_NET,subnet=${RDMA_NET}-sub-3,no-address \
--metadata-from-file=startup-script=tlinux4-customer-init.sh
--local-ssd — A4X 自动挂载 12TB NVMe Local SSD--provisioning-model=RESERVATION_BOUND--resource-policies (placement policy) 以分配到正确的 NVL72 Domainno-address--local-ssd — A4X 自动挂载 12TB NVMe Local SSD--provisioning-model=RESERVATION_BOUND--resource-policies (placement policy) 以分配到正确的 NVL72 Domainno-addressgcloud compute instances create tlinux-test-vm \
--project="$PROJECT_ID" --zone="$A4X_ZONE" \
--machine-type=a4x-highgpu-4g \
--image="$IMAGE_NAME" --image-project="$PROJECT_ID" \
--boot-disk-size=200GB --boot-disk-type=hyperdisk-balanced \
--scopes=cloud-platform \
--reservation-affinity=specific --reservation="$RESERVATION" \
--provisioning-model=RESERVATION_BOUND \
--instance-termination-action=STOP \
--maintenance-policy=TERMINATE \
--restart-on-failure \
--resource-policies="$PLACEMENT_POLICY" \
--network-interface=nic-type=GVNIC,network=$GVNIC_NET,subnet=$GVNIC_SUB \
--network-interface=nic-type=GVNIC,network=$GVNIC_NET_1,subnet=$GVNIC_SUB_1,no-address \
--network-interface=nic-type=MRDMA,network=$RDMA_NET,subnet=${RDMA_NET}-sub-0,no-address \
--network-interface=nic-type=MRDMA,network=$RDMA_NET,subnet=${RDMA_NET}-sub-1,no-address \
--network-interface=nic-type=MRDMA,network=$RDMA_NET,subnet=${RDMA_NET}-sub-2,no-address \
--network-interface=nic-type=MRDMA,network=$RDMA_NET,subnet=${RDMA_NET}-sub-3,no-address
A4X 启动约需 2-3 分钟 (PCI 枚举 GPU/NVSwitch)。先通过串口日志确认 GVE 加载后再 SSH。
# 检查 GVE 和 Guest Agent
gcloud compute instances get-serial-port-output tlinux-test-vm \
--project="$PROJECT_ID" --zone="$A4X_ZONE" 2>&1 | \
grep -i 'gve\|enp0s3\|google_guest_agent'
# 期望: "GVE version 1.4.10-oot" + "DQO RDA queue format"
# 确保 IAP SSH 防火墙规则存在
gcloud compute firewall-rules create allow-iap-ssh \
--project="$PROJECT_ID" \
--network="$GVNIC_NET" \
--allow=tcp:22 \
--source-ranges=35.235.240.0/20 2>/dev/null || true
# SSH via IAP
gcloud compute ssh tlinux-test-vm \
--project="$PROJECT_ID" --zone="$A4X_ZONE" \
--tunnel-through-iap
# 内核
uname -r
# 期望: 6.6.119-49.101.nvidia.tl4.aarch64
# GPU: 4x GB200, 189GB HBM3e
nvidia-smi -L
nvidia-smi --query-gpu=driver_version,gpu_name,memory.total --format=csv,noheader
# 期望: 580.159.04, NVIDIA GB200, 189471 MiB (x4)
# 模块类型 (Open)
lsmod | grep nvidia
# nvidia_peermem 不可用属正常 (A4X 使用 GIB plugin)
# GVE 版本
modinfo gve | grep version
# 期望: version: 1.4.10-oot
# 网卡: 2x GVNIC (ip link 可见) + 4x MRDMA (仅 rdma link 可见,无 Linux 网络接口)
ip -brief addr show
# 期望: enp0s3 (GVNIC), enp192s2 (GVNIC-1)
# 注意: MRDMA VF 为纯 RDMA 设备,不创建 Linux 网络接口,ip link 中不会显示
# RDMA 链路 (验证 RDMA 设备)
rdma link show
# 期望: mlx5_0..3/1 state ACTIVE physical_state LINK_UP
# 验证 RDMA VF 无 netdev (预期为空)
for i in 0 1 2 3; do
echo "mlx5_$i netdev: $(ls /sys/class/infiniband/mlx5_$i/device/net/ 2>/dev/null || echo NONE)"
done
# GCE Guest Agent
systemctl is-active google-guest-agent # 期望: active
# IMEX
systemctl is-enabled nvidia-imex # 期望: enabled
ls /dev/nvidia-caps-imex-channels/ # 期望: channel0
# GDRCopy
systemctl is-active gdrcopy # 期望: active
modinfo gdrdrv | head -3
# SELinux & 防火墙
getenforce # 期望: Disabled
systemctl is-active firewalld # 期望: inactive
| 检查项 | 期望结果 |
|---|---|
| Kernel | 6.6.119-49.101.nvidia.tl4.aarch64 |
| NVIDIA Driver | 580.159.04 (Open, Dual MIT/GPL) |
| GPU | 4x NVIDIA GB200, 189471 MiB HBM3e |
| GVE | v1.4.10-oot, DQO RDA |
| IDPF | v1.0.11 (模块已安装, GB300 可用) |
| RDMA | 4x mlx5 ACTIVE |
| IMEX | enabled, channel0 (GPU-conditional) |
| GDRCopy | v2.6, gdrdrv loaded, service active (GPU-conditional) |
| google-guest-agent | active |
| SSH via IAP | 正常 |
| 非 GPU 实例 | gdrcopy/nvidia-imex 自动跳过, 无 failed 服务 |
gcloud compute instances stop tlinux-test-vm --zone=$A4X_ZONE --project=$PROJECT_ID --discard-local-ssd=true
此脚本适用于 v5.3 镜像,作为 GCE VM 的 startup-script 使用。v5.3 镜像已内置 NVIDIA driver、GVE、IMEX、GDRCopy、nvidia-persistenced 等所有组件,startup script 仅负责文件系统初始化、sshd 56000 端口和 RDMA IB 设备改名。
tlinux4-customer-init.sh),在创建 VM 时通过 --metadata-from-file=startup-script=tlinux4-customer-init.sh 传入。或者在 GCE Console 中粘贴到"Startup script"字段。
#!/bin/bash
# TLinux 4 (TencentOS Server 4.0) — v5.3 Image Startup Script
# Compatible with: A4X/GB200, GB300, H100, CPU-only instances
#
# v5.3 镜像已内置: NVIDIA driver, GVE, IMEX, GDRCopy, nvidia-persistenced,
# SELinux/firewalld disabled, memlock unlimited, sudoers, SSH
#
# 本脚本仅负责: 文件系统初始化 + sshd 56000 端口 + RDMA IB 设备改名
set -euo pipefail
exec > >(tee /var/log/tlinux4-customer-init.log) 2>&1
INIT_STAMP=/etc/.customer-init-done
if [ -f "$INIT_STAMP" ]; then
echo "Customer initialization already completed. Skipping."
exit 0
fi
echo "================================================================"
echo " TLinux 4 v5.3 — Startup Script"
echo " $(date)"
echo "================================================================"
###############################################################################
# [1] Boot disk partitioning: /usr/local (20G) + /data (remaining)
###############################################################################
echo "=== [1] Boot disk partitioning ==="
BOOT_DEV=$(findmnt -n -o SOURCE / | sed 's/p[0-9]*$//')
PARTITION_STAMP=/etc/.partitions-created
if [ ! -f "$PARTITION_STAMP" ]; then
echo "--- Creating boot disk partitions (1TB disk) ---"
sgdisk -n 3:0:+20G -t 3:8300 "$BOOT_DEV"
sgdisk -n 4:0:0 -t 4:8300 "$BOOT_DEV"
partprobe "$BOOT_DEV" 2>/dev/null || true
sleep 2
mkfs.xfs -f "${BOOT_DEV}p3"
mkfs.xfs -f "${BOOT_DEV}p4"
TMPMNT=$(mktemp -d)
mount "${BOOT_DEV}p3" "$TMPMNT"
cp -a /usr/local/. "$TMPMNT/" 2>/dev/null || true
umount "$TMPMNT"
rmdir "$TMPMNT"
mount "${BOOT_DEV}p3" /usr/local
mkdir -p /data
mount "${BOOT_DEV}p4" /data
P3_UUID=$(blkid -s UUID -o value "${BOOT_DEV}p3")
P4_UUID=$(blkid -s UUID -o value "${BOOT_DEV}p4")
if [ -n "$P3_UUID" ] && ! grep -q "$P3_UUID" /etc/fstab; then
echo "UUID=$P3_UUID /usr/local xfs defaults,nofail 0 2" >> /etc/fstab
fi
if [ -n "$P4_UUID" ] && ! grep -q "$P4_UUID" /etc/fstab; then
echo "UUID=$P4_UUID /data xfs defaults,nofail 0 2" >> /etc/fstab
fi
touch "$PARTITION_STAMP"
echo "Boot disk partitioned: p3=/usr/local(20G) p4=/data(remaining)"
fi
###############################################################################
# [2] Local SSD RAID0 → /mnt/stateful_partition
###############################################################################
echo "=== [2] Local SSD RAID0 setup ==="
SSD_DEVICES=()
for disk in /dev/nvme*n1; do
[ -b "$disk" ] || continue
PART_COUNT=$(lsblk -n -o NAME "$disk" 2>/dev/null | wc -l)
if [ "$PART_COUNT" -le 1 ]; then
SSD_DEVICES+=("$disk")
echo " Detected local SSD: $disk ($(lsblk -n -d -o SIZE "$disk"))"
fi
done
NUM_SSDS=${#SSD_DEVICES[@]}
echo " Found $NUM_SSDS local SSD device(s)"
if [ "$NUM_SSDS" -gt 0 ]; then
MOUNT_POINT="/mnt/stateful_partition"
mkdir -p "$MOUNT_POINT"
if [ "$NUM_SSDS" -eq 1 ]; then
TARGET_DEV="${SSD_DEVICES[0]}"
mkfs.xfs -f "$TARGET_DEV"
mount "$TARGET_DEV" "$MOUNT_POINT"
DEV_UUID=$(blkid -s UUID -o value "$TARGET_DEV")
if [ -n "$DEV_UUID" ] && ! grep -q "$DEV_UUID" /etc/fstab; then
echo "UUID=$DEV_UUID $MOUNT_POINT xfs defaults,nofail 0 2" >> /etc/fstab
fi
else
mdadm --stop /dev/md0 2>/dev/null || true
mdadm --create /dev/md0 --level=0 --raid-devices="$NUM_SSDS" "${SSD_DEVICES[@]}" --force --run
mkfs.xfs -f /dev/md0
mount /dev/md0 "$MOUNT_POINT"
mkdir -p /etc/mdadm
mdadm --detail --scan >> /etc/mdadm/mdadm.conf
DEV_UUID=$(blkid -s UUID -o value /dev/md0)
if [ -n "$DEV_UUID" ] && ! grep -q "$DEV_UUID" /etc/fstab; then
echo "UUID=$DEV_UUID $MOUNT_POINT xfs defaults,nofail 0 2" >> /etc/fstab
fi
fi
mkdir -p "$MOUNT_POINT"/{gcsfuse-cache,scratch-data}
echo "$MOUNT_POINT mounted (total: $(df -h "$MOUNT_POINT" | awk 'NR==2{print $2}'))"
else
echo "WARNING: No local SSD devices found"
mkdir -p /mnt/stateful_partition
fi
###############################################################################
# [3] sshd port 56000
###############################################################################
echo "=== [3] sshd port 56000 ==="
if ! grep -q '^Port 56000' /etc/ssh/sshd_config; then
sed -i 's/^Port 22$/Port 22\nPort 56000/' /etc/ssh/sshd_config
systemctl restart sshd
echo "sshd now listening on port 22 and 56000"
else
echo "sshd port 56000 already configured"
fi
###############################################################################
# [4] RDMA IB device rename (bond1-bond4)
###############################################################################
echo "=== [4] RDMA IB device rename ==="
for i in 0 1 2 3; do
if rdma dev set mlx5_${i} name bond$((i+1)) 2>/dev/null; then
echo " mlx5_${i} → bond$((i+1))"
fi
done
rdma link show 2>/dev/null || echo " No RDMA devices (non-GPU instance)"
###############################################################################
# Verification
###############################################################################
echo ""
echo "================================================================"
echo " Verification"
echo "================================================================"
echo "--- OS ---"
cat /etc/os-release | head -3
echo "Kernel: $(uname -r)"
if lspci 2>/dev/null | grep -qi 'NVIDIA'; then
echo ""
echo "--- GPU ---"
nvidia-smi -L 2>&1 | head -4
echo "Persistence: $(nvidia-smi -q 2>/dev/null | grep 'Persistence Mode' | head -1 || echo N/A)"
echo "IMEX: $(ls /dev/nvidia-caps-imex-channels/ 2>&1)"
fi
echo ""
echo "--- Network ---"
echo "RDMA: $(rdma link show 2>/dev/null | awk '{print $2}' | tr '\n' ', ' || echo N/A)"
echo ""
echo "--- Disk ---"
df -h /usr/local /data /mnt/stateful_partition 2>/dev/null | grep -v Filesystem
echo ""
echo "--- SSH ---"
echo "Ports: $(grep '^Port' /etc/ssh/sshd_config | awk '{print $2}' | tr '\n' ', ')"
touch "$INIT_STAMP"
echo ""
echo "================================================================"
echo " Startup complete at $(date)"
echo "================================================================"