#!/bin/bash
# TLinux 4 (TencentOS Server 4.0) — v5.3 Image Internal Test Environment
# Includes k8s worker setup (containerd + nvidia-ctk + kubeadm join)
#
# v5.3 镜像已内置: NVIDIA driver, GVE, IMEX, GDRCopy, nvidia-persistenced,
#                   SELinux/firewalld disabled, memlock unlimited, sudoers, SSH
#
# 本脚本负责: 文件系统初始化 + sshd 22/56000 端口 + 网卡改名 (bond0-bond5) + k8s worker
set -euo pipefail
exec > >(tee /var/log/tlinux4-internal-init.log) 2>&1

K8S_VERSION="1.34"
K8S_PATCH="1.34.9"
INIT_STAMP=/etc/.internal-init-done

###############################################################################
# Network interface rename (A4X only: bond0-bond5) — runs every boot
# GVNIC: enp0s3→bond0, enp192s2→bond1 (persisted via udev after first boot)
# RDMA: mlx5_0→bond2, mlx5_1→bond3, mlx5_2→bond4, mlx5_3→bond5
###############################################################################
if rdma link show 2>/dev/null | grep -q .; then
  echo "=== Network interface rename (A4X) ==="

  # GVNIC rename (Linux netdev: ip link set name)
  for pair in "enp0s3:bond0" "enp192s2:bond1"; do
    OLD="${pair%%:*}"
    NEW="${pair##*:}"
    if ip link show "$OLD" &>/dev/null; then
      ip link set dev "$OLD" down
      ip link set dev "$OLD" name "$NEW"
      ip link set dev "$NEW" up
      echo "  $OLD → $NEW"
    elif ip link show "$NEW" &>/dev/null; then
      echo "  $NEW already exists (skip)"
    fi
  done

  # Update NM connection profile to match new name
  NM_CONN="/etc/NetworkManager/system-connections/enp0s3-gvnic.nmconnection"
  if [ -f "$NM_CONN" ] && grep -q "interface-name=enp0s3" "$NM_CONN"; then
    sed -i 's/interface-name=enp0s3/interface-name=bond0/' "$NM_CONN"
    sed -i 's/id=enp0s3-gvnic/id=bond0-gvnic/' "$NM_CONN"
    nmcli connection reload 2>/dev/null || true
    echo "  Updated NM profile: enp0s3-gvnic → bond0-gvnic"
  fi

  # Create udev rule for persistent GVNIC naming across reboots (one-time)
  UDEV_RULE="/etc/udev/rules.d/70-a4x-bond-rename.rules"
  if [ ! -f "$UDEV_RULE" ]; then
    PCI0=$(basename "$(readlink -f /sys/class/net/bond0/device)")
    PCI1=$(basename "$(readlink -f /sys/class/net/bond1/device)")
    cat > "$UDEV_RULE" << UDEVEOF
SUBSYSTEM=="net", ACTION=="add", KERNELS=="$PCI0", NAME="bond0"
SUBSYSTEM=="net", ACTION=="add", KERNELS=="$PCI1", NAME="bond1"
UDEVEOF
    echo "  Created udev rule for persistent GVNIC naming"
  fi

  # RDMA IB device rename (rdma dev set — runtime only, needs every boot)
  for i in 0 1 2 3; do
    if rdma dev set mlx5_${i} name bond$((i+2)) 2>/dev/null; then
      echo "  mlx5_${i} → bond$((i+2))"
    fi
  done

  echo "  --- Result ---"
  ip -br link show type ether 2>/dev/null || true
  rdma link show 2>/dev/null || true
fi

# v5.3 image has stale stamp from build; remove if partitions not yet created
if [ -f "$INIT_STAMP" ] && [ ! -f "/etc/.partitions-created" ]; then
  echo "Removing stale init stamp from image build..."
  rm -f "$INIT_STAMP"
fi

if [ -f "$INIT_STAMP" ]; then
  echo "Internal initialization already completed. Skipping."
  exit 0
fi

echo "================================================================"
echo "  TLinux 4 v5.3 — Internal Test Startup Script"
echo "  $(date)"
echo "================================================================"

###############################################################################
# [1] Boot disk partitioning: /usr/local (20G) + /data (remaining)
###############################################################################
echo "=== [1] Boot disk partitioning ==="

BOOT_DEV=$(findmnt -n -o SOURCE / | sed 's/p[0-9]*$//')
PARTITION_STAMP=/etc/.partitions-created

if [ ! -f "$PARTITION_STAMP" ]; then
  echo "--- Creating boot disk partitions (1TB disk) ---"

  # Expand GPT to full disk (image GPT backup may be at original image size)
  sgdisk -e "$BOOT_DEV" 2>/dev/null || true

  sgdisk -n 3:0:+20G -t 3:8300 "$BOOT_DEV"
  sgdisk -n 4:0:0 -t 4:8300 "$BOOT_DEV"
  partprobe "$BOOT_DEV" 2>/dev/null || true
  sleep 2

  mkfs.xfs -f "${BOOT_DEV}p3"
  mkfs.xfs -f "${BOOT_DEV}p4"

  TMPMNT=$(mktemp -d)
  mount "${BOOT_DEV}p3" "$TMPMNT"
  cp -a /usr/local/. "$TMPMNT/" 2>/dev/null || true
  umount "$TMPMNT"
  rmdir "$TMPMNT"

  mount "${BOOT_DEV}p3" /usr/local
  mkdir -p /data
  mount "${BOOT_DEV}p4" /data

  P3_UUID=$(blkid -s UUID -o value "${BOOT_DEV}p3")
  P4_UUID=$(blkid -s UUID -o value "${BOOT_DEV}p4")
  if [ -n "$P3_UUID" ] && ! grep -q "$P3_UUID" /etc/fstab; then
    echo "UUID=$P3_UUID /usr/local xfs defaults,nofail 0 2" >> /etc/fstab
  fi
  if [ -n "$P4_UUID" ] && ! grep -q "$P4_UUID" /etc/fstab; then
    echo "UUID=$P4_UUID /data xfs defaults,nofail 0 2" >> /etc/fstab
  fi

  touch "$PARTITION_STAMP"
  echo "Boot disk partitioned: p3=/usr/local(20G) p4=/data(remaining)"
fi

###############################################################################
# [2] Local SSD RAID0 → /mnt/stateful_partition
###############################################################################
echo "=== [2] Local SSD RAID0 setup ==="

SSD_DEVICES=()
for disk in /dev/nvme*n1; do
  [ -b "$disk" ] || continue
  PART_COUNT=$(lsblk -n -o NAME "$disk" 2>/dev/null | wc -l)
  if [ "$PART_COUNT" -le 1 ]; then
    SSD_DEVICES+=("$disk")
    echo "  Detected local SSD: $disk ($(lsblk -n -d -o SIZE "$disk"))"
  fi
done

NUM_SSDS=${#SSD_DEVICES[@]}
echo "  Found $NUM_SSDS local SSD device(s)"

if [ "$NUM_SSDS" -gt 0 ]; then
  MOUNT_POINT="/mnt/stateful_partition"
  mkdir -p "$MOUNT_POINT"

  if [ "$NUM_SSDS" -eq 1 ]; then
    TARGET_DEV="${SSD_DEVICES[0]}"
    mkfs.xfs -f "$TARGET_DEV"
    mount "$TARGET_DEV" "$MOUNT_POINT"
    DEV_UUID=$(blkid -s UUID -o value "$TARGET_DEV")
    if [ -n "$DEV_UUID" ] && ! grep -q "$DEV_UUID" /etc/fstab; then
      echo "UUID=$DEV_UUID $MOUNT_POINT xfs defaults,nofail 0 2" >> /etc/fstab
    fi
  else
    mdadm --stop /dev/md0 2>/dev/null || true
    mdadm --create /dev/md0 --level=0 --raid-devices="$NUM_SSDS" "${SSD_DEVICES[@]}" --force --run
    mkfs.xfs -f /dev/md0
    mount /dev/md0 "$MOUNT_POINT"
    mkdir -p /etc/mdadm
    mdadm --detail --scan >> /etc/mdadm/mdadm.conf
    DEV_UUID=$(blkid -s UUID -o value /dev/md0)
    if [ -n "$DEV_UUID" ] && ! grep -q "$DEV_UUID" /etc/fstab; then
      echo "UUID=$DEV_UUID $MOUNT_POINT xfs defaults,nofail 0 2" >> /etc/fstab
    fi
  fi

  mkdir -p "$MOUNT_POINT"/{gcsfuse-cache,scratch-data}
  echo "$MOUNT_POINT mounted (total: $(df -h "$MOUNT_POINT" | awk 'NR==2{print $2}'))"
else
  echo "WARNING: No local SSD devices found"
  mkdir -p /mnt/stateful_partition
fi

###############################################################################
# [3] sshd port 22 + 56000
###############################################################################
echo "=== [3] sshd port 22 + 56000 ==="

# TLinux 4 base image uses Port 36000 with Port 22 commented out.
# Ensure both Port 22 and Port 56000 are explicitly configured.
SSHD_CHANGED=0
if ! grep -q '^Port 22' /etc/ssh/sshd_config; then
  sed -i '1i Port 22' /etc/ssh/sshd_config
  SSHD_CHANGED=1
fi
if ! grep -q '^Port 56000' /etc/ssh/sshd_config; then
  sed -i '/^Port 22/a Port 56000' /etc/ssh/sshd_config
  SSHD_CHANGED=1
fi
if [ "$SSHD_CHANGED" -eq 1 ]; then
  systemctl restart sshd
  echo "sshd now listening on port 22 and 56000"
else
  echo "sshd ports 22 and 56000 already configured"
fi

###############################################################################
# [4] k8s worker setup (containerd + nvidia-ctk + kubeadm join)
###############################################################################
echo "=== [4] k8s $K8S_VERSION worker setup ==="

# [4.1] Swap off
echo "--- [4.1] Disable swap ---"
swapoff -a
sed -i '/swap/d' /etc/fstab

# [4.2] Kernel modules for k8s
echo "--- [4.2] Kernel modules ---"
cat > /etc/modules-load.d/k8s.conf <<EOF
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter
cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sysctl --system

# [4.3] Install containerd
echo "--- [4.3] containerd ---"
dnf install -y --enablerepo=AppStream container-selinux 2>/dev/null || true
DOCKER_ARCH=$(uname -m)
cat > /etc/yum.repos.d/docker-ce.repo <<REPOEOF
[docker-ce-stable]
name=Docker CE Stable - ${DOCKER_ARCH}
baseurl=https://download.docker.com/linux/rhel/9/${DOCKER_ARCH}/stable
enabled=1
gpgcheck=1
gpgkey=https://download.docker.com/linux/rhel/gpg
REPOEOF
dnf install -y containerd.io
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
mkdir -p /data/containerd
systemctl enable --now containerd

# [4.4] NVIDIA container toolkit
echo "--- [4.4] NVIDIA container toolkit ---"
curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo | \
  tee /etc/yum.repos.d/nvidia-container-toolkit.repo
dnf install -y nvidia-container-toolkit
nvidia-ctk runtime configure --runtime=containerd --set-as-default
# containerd config default uses single quotes; nvidia-ctk may rewrite with double quotes
sed -i "s|root = .*/var/lib/containerd.*|root = \"/data/containerd\"|" /etc/containerd/config.toml
systemctl restart containerd

# [4.5] kubeadm / kubelet / kubectl
echo "--- [4.5] kubeadm/kubelet/kubectl ---"
cat > /etc/yum.repos.d/kubernetes.repo <<REPOEOF
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v${K8S_VERSION}/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v${K8S_VERSION}/rpm/repodata/repomd.xml.key
REPOEOF
dnf install -y --enablerepo=AppStream kubelet-${K8S_PATCH} kubeadm-${K8S_PATCH} kubectl-${K8S_PATCH} conntrack-tools socat iptables ethtool
systemctl enable kubelet

# [4.6] kubeadm join
echo "--- [4.6] kubeadm join ---"
CP_IP=$(curl -sf -H "Metadata-Flavor: Google" \
  http://metadata.google.internal/computeMetadata/v1/instance/attributes/cp-ip 2>/dev/null || true)
JOIN_TOKEN=$(curl -sf -H "Metadata-Flavor: Google" \
  http://metadata.google.internal/computeMetadata/v1/instance/attributes/join-token 2>/dev/null || true)
JOIN_HASH=$(curl -sf -H "Metadata-Flavor: Google" \
  http://metadata.google.internal/computeMetadata/v1/instance/attributes/join-hash 2>/dev/null || true)

NODE_NAME=$(cat /proc/sys/kernel/hostname 2>/dev/null || curl -sf -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/name || echo "unknown")
if [[ -n "$CP_IP" && -n "$JOIN_TOKEN" && -n "$JOIN_HASH" ]]; then
  kubeadm join ${CP_IP}:6443 \
    --token "${JOIN_TOKEN}" \
    --discovery-token-ca-cert-hash "sha256:${JOIN_HASH}" \
    --node-name "${NODE_NAME}" \
    --ignore-preflight-errors=Hostname 2>&1 || echo "kubeadm join failed, manual join required"
else
  echo "Missing CP metadata (cp-ip, join-token, join-hash). Manual join required."
  echo "Run on CP: kubeadm token create --print-join-command"
fi

###############################################################################
# Verification
###############################################################################
echo ""
echo "================================================================"
echo "  Verification"
echo "================================================================"

echo "--- OS ---"
cat /etc/os-release | head -3
echo "Kernel: $(uname -r)"

if lspci 2>/dev/null | grep -qi 'NVIDIA'; then
  echo ""
  echo "--- GPU ---"
  nvidia-smi -L 2>&1 | head -4
  echo "Persistence: $(nvidia-smi -q 2>/dev/null | grep 'Persistence Mode' | head -1 || echo N/A)"
  echo "IMEX: $(ls /dev/nvidia-caps-imex-channels/ 2>&1)"
fi

echo ""
echo "--- Network ---"
ip -br link show type ether 2>/dev/null || true
echo "RDMA: $(rdma link show 2>/dev/null | awk '{print $2}' | tr '\n' ', ' || echo N/A)"

echo ""
echo "--- Services ---"
for svc in containerd kubelet; do
  echo "$svc: $(systemctl is-active $svc 2>/dev/null || echo inactive)"
done

echo ""
echo "--- Disk ---"
df -h /usr/local /data /mnt/stateful_partition 2>/dev/null | grep -v Filesystem

echo ""
echo "--- SSH ---"
echo "Ports: $(grep '^Port' /etc/ssh/sshd_config | awk '{print $2}' | tr '\n' ', ')"

echo ""
echo "--- k8s ---"
kubeadm version 2>/dev/null || echo "kubeadm not installed"

touch "$INIT_STAMP"
echo ""
echo "================================================================"
echo "  Internal initialization complete at $(date)"
echo "================================================================"
