#!/bin/bash
# TLinux 4 (TencentOS Server 4.0) — GB300 (A4X-Max) Internal Test Startup Script
# Compatible with: a4x-maxgpu-4g-metal (Grace Blackwell Bare Metal)
#
# Automatically installs: containerd, nvidia-container-toolkit, and K8s Worker components
# then joins the node to the self-managed K8s cluster.
#
set -euo pipefail
exec > >(tee /var/log/tlinux4-internal-init.log) 2>&1

K8S_VERSION="1.34"
K8S_PATCH="1.34.9"
INTERNAL_INIT_STAMP=/etc/.internal-init-done

# 1. Run the customer baseline initialization script to setup disk, naming and routing
SCRIPT_DIR="$(dirname "$0")"
if [ -f "$SCRIPT_DIR/tlinux4-customer-init-gb300.sh" ]; then
  echo "=== Running customer-init base setup ==="
  bash "$SCRIPT_DIR/tlinux4-customer-init-gb300.sh"
else
  echo "ERROR: tlinux4-customer-init-gb300.sh not found in $SCRIPT_DIR!"
  exit 1
fi

if [ -f "$INTERNAL_INIT_STAMP" ]; then
  echo "Internal test environment setup already completed. Skipping."
  exit 0
fi

echo "================================================================"
echo "  TLinux 4 v5.4 — GB300 Internal Test Startup Script"
echo "  $(date)"
echo "================================================================"

###############################################################################
# [1] K8s Kernel parameters & module settings
###############################################################################
echo "=== [1] Configuring Kernel Modules for Kubernetes ==="

cat > /etc/modules-load.d/k8s.conf <<EOF
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter

cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sysctl --system

###############################################################################
# [2] Install containerd container runtime
###############################################################################
echo "=== [2] Installing containerd ==="

dnf install -y --enablerepo=AppStream container-selinux 2>/dev/null || true
DOCKER_ARCH=$(uname -m)

cat > /etc/yum.repos.d/docker-ce.repo <<REPOEOF
[docker-ce-stable]
name=Docker CE Stable - ${DOCKER_ARCH}
baseurl=https://download.docker.com/linux/rhel/9/${DOCKER_ARCH}/stable
enabled=1
gpgcheck=1
gpgkey=https://download.docker.com/linux/rhel/gpg
REPOEOF

dnf install -y containerd.io
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

# Redirect containerd root directory to /data/containerd to utilize SSD/HDD space
mkdir -p /data/containerd
sed -i "s|root = .*/var/lib/containerd.*|root = \"/data/containerd\"|" /etc/containerd/config.toml

systemctl enable --now containerd

###############################################################################
# [3] Install NVIDIA Container Toolkit
###############################################################################
echo "=== [3] Installing NVIDIA Container Toolkit ==="

curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo | \
  tee /etc/yum.repos.d/nvidia-container-toolkit.repo

dnf install -y nvidia-container-toolkit
nvidia-ctk runtime configure --runtime=containerd --set-as-default

systemctl restart containerd

###############################################################################
# [4] Install Kubeadm, Kubelet and Kubelet CLI tools
###############################################################################
echo "=== [4] Installing K8s packages (kubeadm/kubelet/kubectl) ==="

cat > /etc/yum.repos.d/kubernetes.repo <<REPOEOF
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v${K8S_VERSION}/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v${K8S_VERSION}/rpm/repodata/repomd.xml.key
REPOEOF

dnf install -y --enablerepo=AppStream kubelet-${K8S_PATCH} kubeadm-${K8S_PATCH} kubectl-${K8S_PATCH} conntrack-tools socat iptables ethtool
systemctl enable kubelet

###############################################################################
# [5] Kubeadm Join (via GCE Instance Metadata)
###############################################################################
echo "=== [5] Kubeadm Join ==="

CP_IP=$(curl -sf -H "Metadata-Flavor: Google" \
  http://metadata.google.internal/computeMetadata/v1/instance/attributes/cp-ip 2>/dev/null || true)
JOIN_TOKEN=$(curl -sf -H "Metadata-Flavor: Google" \
  http://metadata.google.internal/computeMetadata/v1/instance/attributes/join-token 2>/dev/null || true)
JOIN_HASH=$(curl -sf -H "Metadata-Flavor: Google" \
  http://metadata.google.internal/computeMetadata/v1/instance/attributes/join-hash 2>/dev/null || true)

NODE_NAME=$(cat /proc/sys/kernel/hostname 2>/dev/null || curl -sf -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/name || echo "unknown")

if [[ -n "$CP_IP" && -n "$JOIN_TOKEN" && -n "$JOIN_HASH" ]]; then
  echo "  Joining cluster at ${CP_IP}:6443..."
  kubeadm join ${CP_IP}:6443 \
    --token "${JOIN_TOKEN}" \
    --discovery-token-ca-cert-hash "sha256:${JOIN_HASH}" \
    --node-name "${NODE_NAME}" \
    --ignore-preflight-errors=Hostname 2>&1 || echo "kubeadm join failed, manual join required"
else
  echo "  Missing CP metadata parameters (cp-ip, join-token, join-hash). Skipping auto-join."
  echo "  Please execute manual join on the node. Retrieve token command on K8s Control Plane:"
  echo "  kubeadm token create --print-join-command"
fi

###############################################################################
# Verification
###############################################################################
echo ""
echo "================================================================"
echo "  Verification Results"
echo "================================================================"
echo "Containerd: $(systemctl is-active containerd 2>/dev/null)"
echo "Kubelet: $(systemctl is-active kubelet 2>/dev/null)"
kubeadm version 2>/dev/null || echo "kubeadm not installed"

touch "$INTERNAL_INIT_STAMP"
echo "=== Internal Test Initialization Complete ==="
