#!/bin/bash
# TLinux 4 (TencentOS Server 4.0) — GB300 (A4X-Max) Startup Script (Customer Version)
# Compatible with: a4x-maxgpu-4g-metal (Grace Blackwell Bare Metal)
#
# Configures: 
#   1. Boot Disk Partitions (/usr/local 20G, /data remaining)
#   2. Local SSD RAID0 (/mnt/stateful_partition, 4x 3TB = 12TB)
#   3. Network Interface Renaming:
#      - IDPF netdevs: eth0 -> bond0, eth1 -> bond1
#      - MRDMA ibdevs: mlx5_0..7 -> bond2..9
#   4. Dynamic NUMA-aligned 2MB Hugepages Allocation (16GB total)
#   5. Lustre storage interface routing (eth1 / bond1)
#   6. SSH ports 22 + 56000 configuration
#
set -euo pipefail
exec > >(tee /var/log/tlinux4-customer-init.log) 2>&1

INIT_STAMP=/etc/.customer-init-done

###############################################################################
# Network interface rename (IDPF & MRDMA) — runs every boot
# IDPF: eth0→bond0 (mgmt), eth1→bond1 (lustre client)
# MRDMA: mlx5_0..7→bond2..9
###############################################################################
echo "=== Network interface rename (GB300) ==="

# 1. IDPF Interface renaming & persistent udev rules creation (one-time)
UDEV_RULE="/etc/udev/rules.d/70-idpf-bond-rename.rules"
if [ ! -f "$UDEV_RULE" ]; then
  echo "  Creating udev rule for persistent IDPF naming..."
  # Detect IDPF PCI devices in sorting order
  PCI_DEVS=($(ls -la /sys/bus/pci/drivers/idpf/ | grep '\->' | awk -F'/' '{print $NF}' | sort))
  if [ "${#PCI_DEVS[@]}" -ge 2 ]; then
    PCI0="${PCI_DEVS[0]}"
    PCI1="${PCI_DEVS[1]}"
    
    cat > "$UDEV_RULE" << UDEVEOF
SUBSYSTEM=="net", ACTION=="add", KERNELS=="$PCI0", NAME="bond0"
SUBSYSTEM=="net", ACTION=="add", KERNELS=="$PCI1", NAME="bond1"
UDEVEOF
    echo "  udev rules written: $PCI0 -> bond0, $PCI1 -> bond1"
    
    # Apply rename immediately for first boot
    for dev in eth0 eth1; do
      if ip link show "$dev" &>/dev/null; then
        PCI_DEV=$(basename "$(readlink -f /sys/class/net/$dev/device)")
        if [ "$PCI_DEV" = "$PCI0" ]; then
          ip link set dev "$dev" down
          ip link set dev "$dev" name bond0
          ip link set dev bond0 up
          echo "  Immediate rename: $dev -> bond0"
        elif [ "$PCI_DEV" = "$PCI1" ]; then
          ip link set dev "$dev" down
          ip link set dev "$dev" name bond1
          ip link set dev bond1 up
          echo "  Immediate rename: $dev -> bond1"
        fi
      fi
    done
  else
    echo "  ERROR: Less than 2 IDPF interfaces found. Skipping IDPF rename."
  fi
else
  echo "  udev rule $UDEV_RULE already exists (IDPF named persistently)"
fi

# 2. Update NetworkManager profiles to bind to new interface names
for idx in 0 1; do
  NM_CONN="/etc/NetworkManager/system-connections/eth${idx}-idpf.nmconnection"
  if [ -f "$NM_CONN" ]; then
    sed -i "s/interface-name=eth${idx}/interface-name=bond${idx}/" "$NM_CONN"
    sed -i "s/id=eth${idx}-idpf/id=bond${idx}-idpf/" "$NM_CONN"
    nmcli connection reload 2>/dev/null || true
    echo "  Updated NM profile: eth${idx} → bond${idx}"
  fi
done

# 3. RDMA IB device renaming (rdma dev set — runtime only, needs to run on every boot)
for i in {0..7}; do
  if rdma dev set "mlx5_${i}" name "bond$((i+2))" 2>/dev/null; then
    echo "  Mellanox IB: mlx5_${i} → bond$((i+2))"
  fi
done

echo "  --- Rename Results ---"
ip -br link show type ether 2>/dev/null || true
rdma link show 2>/dev/null || true


if [ -f "$INIT_STAMP" ]; then
  echo "Customer initialization already completed. Skipping stamp checks."
  exit 0
fi

echo "================================================================"
# TLinux 4 v5.4 initialization
echo "  TLinux 4 v5.4 — GB300 Customer Startup Script"
echo "  $(date)"
echo "================================================================"

###############################################################################
# [1] Boot disk partitioning: /usr/local (20G) + /data (remaining)
###############################################################################
echo "=== [1] Boot disk partitioning ==="

BOOT_DEV=$(findmnt -n -o SOURCE / | sed 's/p[0-9]*$//')
PARTITION_STAMP=/etc/.partitions-created

if [ ! -f "$PARTITION_STAMP" ]; then
  echo "--- Creating boot disk partitions (1TB disk) ---"
  # Expand GPT to full disk
  sgdisk -e "$BOOT_DEV" 2>/dev/null || true

  sgdisk -n 3:0:+20G -t 3:8300 "$BOOT_DEV"
  sgdisk -n 4:0:0 -t 4:8300 "$BOOT_DEV"
  partprobe "$BOOT_DEV" 2>/dev/null || true
  sleep 2

  mkfs.xfs -f "${BOOT_DEV}p3"
  mkfs.xfs -f "${BOOT_DEV}p4"

  TMPMNT=$(mktemp -d)
  mount "${BOOT_DEV}p3" "$TMPMNT"
  cp -a /usr/local/. "$TMPMNT/" 2>/dev/null || true
  umount "$TMPMNT"
  rmdir "$TMPMNT"

  mount "${BOOT_DEV}p3" /usr/local
  mkdir -p /data
  mount "${BOOT_DEV}p4" /data

  P3_UUID=$(blkid -s UUID -o value "${BOOT_DEV}p3")
  P4_UUID=$(blkid -s UUID -o value "${BOOT_DEV}p4")
  if [ -n "$P3_UUID" ] && ! grep -q "$P3_UUID" /etc/fstab; then
    echo "UUID=$P3_UUID /usr/local xfs defaults,nofail 0 2" >> /etc/fstab
  fi
  if [ -n "$P4_UUID" ] && ! grep -q "$P4_UUID" /etc/fstab; then
    echo "UUID=$P4_UUID /data xfs defaults,nofail 0 2" >> /etc/fstab
  fi

  touch "$PARTITION_STAMP"
  echo "Boot disk partitioned: p3=/usr/local(20G) p4=/data(remaining)"
fi

###############################################################################
# [2] Local SSD RAID0 setup (4x 3TB NVMe Local SSDs)
###############################################################################
echo "=== [2] Local SSD RAID0 setup ==="

SSD_DEVICES=()
for disk in /dev/nvme*n1; do
  [ -b "$disk" ] || continue
  PART_COUNT=$(lsblk -n -o NAME "$disk" 2>/dev/null | wc -l)
  if [ "$PART_COUNT" -le 1 ]; then
    SSD_DEVICES+=("$disk")
    echo "  Detected local SSD: $disk ($(lsblk -n -d -o SIZE "$disk"))"
  fi
done

NUM_SSDS=${#SSD_DEVICES[@]}
echo "  Found $NUM_SSDS local SSD device(s)"

if [ "$NUM_SSDS" -gt 0 ]; then
  MOUNT_POINT="/mnt/stateful_partition"
  mkdir -p "$MOUNT_POINT"

  if [ "$NUM_SSDS" -eq 1 ]; then
    TARGET_DEV="${SSD_DEVICES[0]}"
    mkfs.xfs -f "$TARGET_DEV"
    mount "$TARGET_DEV" "$MOUNT_POINT"
    DEV_UUID=$(blkid -s UUID -o value "$TARGET_DEV")
    if [ -n "$DEV_UUID" ] && ! grep -q "$DEV_UUID" /etc/fstab; then
      echo "UUID=$DEV_UUID $MOUNT_POINT xfs defaults,nofail 0 2" >> /etc/fstab
    fi
  else
    mdadm --stop /dev/md0 2>/dev/null || true
    mdadm --create /dev/md0 --level=0 --raid-devices="$NUM_SSDS" "${SSD_DEVICES[@]}" --force --run
    mkfs.xfs -f /dev/md0
    mount /dev/md0 "$MOUNT_POINT"
    mkdir -p /etc/mdadm
    mdadm --detail --scan >> /etc/mdadm/mdadm.conf
    DEV_UUID=$(blkid -s UUID -o value /dev/md0)
    if [ -n "$DEV_UUID" ] && ! grep -q "$DEV_UUID" /etc/fstab; then
      echo "UUID=$DEV_UUID $MOUNT_POINT xfs defaults,nofail 0 2" >> /etc/fstab
    fi
  fi

  mkdir -p "$MOUNT_POINT"/{gcsfuse-cache,scratch-data}
  echo "$MOUNT_POINT mounted (total: $(df -h "$MOUNT_POINT" | awk 'NR==2{print $2}'))"
else
  echo "WARNING: No local SSD devices found"
  mkdir -p /mnt/stateful_partition
fi

###############################################################################
# [3] Safe NUMA-aligned 2MB Hugepages allocation (asapd-lite dependency)
###############################################################################
echo "=== [3] Safe Hugepages allocation ==="

# Target 2MB hugepage size explicitly on each NUMA node to prevent OOM hang on 64KB page size OS kernels
for d in /sys/devices/system/node/node[0-9]*; do
  if [ -d "$d/hugepages/hugepages-2048kB" ]; then
    echo 4096 | tee "$d/hugepages/hugepages-2048kB/nr_hugepages"
  fi
done

# Ensure hugetlbfs is mounted
mkdir -p /dev/hugepages
mount -t hugetlbfs none /dev/hugepages 2>/dev/null || true

###############################################################################
# [4] Lustre storage network routing
# Bind LNet networks option to bond1 (MTU 8896 secondary management net)
###############################################################################
echo "=== [4] Lustre Storage routing ==="

echo 'options lnet networks="tcp(bond1)"' > /etc/modprobe.d/lustre.conf

# Add routing policy for Lustre FS subnet via bond1 gateway (typically 10.201.0.1)
LUSTRE_FS_CIDR="172.16.0.0/24"
LUSTRE_GW="10.201.0.1"
ip route add "$LUSTRE_FS_CIDR" via "$LUSTRE_GW" dev bond1 || true

# Persist Lustre route via NM connection dispatcher script
dispatcher_script="/etc/NetworkManager/dispatcher.d/99-lustre-route"
cat > "$dispatcher_script" << 'EOF'
#!/bin/bash
INTERFACE=$1
ACTION=$2
if [ "$INTERFACE" = "bond1" ] && [ "$ACTION" = "up" ]; then
    ip route add 172.16.0.0/24 via 10.201.0.1 dev bond1
fi
EOF
chmod +x "$dispatcher_script"

###############################################################################
# [5] sshd ports 22 + 56000
###############################################################################
echo "=== [5] sshd port 22 + 56000 ==="

SSHD_CHANGED=0
if ! grep -q '^Port 22' /etc/ssh/sshd_config; then
  sed -i '1i Port 22' /etc/ssh/sshd_config
  SSHD_CHANGED=1
fi
if ! grep -q '^Port 56000' /etc/ssh/sshd_config; then
  sed -i '/^Port 22/a Port 56000' /etc/ssh/sshd_config
  SSHD_CHANGED=1
fi
if [ "$SSHD_CHANGED" -eq 1 ]; then
  systemctl restart sshd
  echo "sshd now listening on ports 22 and 56000"
else
  echo "sshd ports 22 and 56000 already configured"
fi

touch "$INIT_STAMP"
echo "=== Customer Initialization Complete ==="
