From 21836c4d7c7909d07c15ec8e591fa77dd83405a5 Mon Sep 17 00:00:00 2001
From: shenlan <manbuzhe2009@qq.com>
Date: Sun, 14 Sep 2025 17:03:41 +0800
Subject: [PATCH 1/3] Fix k3s arm64 binary in offline installer

---
 .../workflows/offline-package-k3s-installer.yaml   | 14 +++++++++++++-
 scripts/make_k3s_offline_package.sh                |  6 +++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/offline-package-k3s-installer.yaml b/.github/workflows/offline-package-k3s-installer.yaml
index 7f7f93f..d4bbda1 100644
--- a/.github/workflows/offline-package-k3s-installer.yaml
+++ b/.github/workflows/offline-package-k3s-installer.yaml
@@ -76,7 +76,7 @@ jobs:
     needs: build-k3s-installer
     strategy:
       matrix:
-        arch: [amd64]
+        arch: [amd64, arm64]
     runs-on: ubuntu-latest
 
     steps:
@@ -91,7 +91,19 @@ jobs:
           cd test-dir
           tar -xzvf offline-package-k3s-installer-${{ matrix.arch }}.tar.gz
 
+      - name: Verify k3s binary arch
+        run: |
+          set -euo pipefail
+          BIN="test-dir/k3s-offline-package/bin/k3s-${{ matrix.arch }}"
+          file "$BIN"
+          if [ "${{ matrix.arch }}" = "amd64" ]; then
+            file "$BIN" | grep -q 'x86-64'
+          else
+            file "$BIN" | grep -q 'aarch64'
+          fi
+
       - name: Setup K3s and Test
+        if: matrix.arch == 'amd64'
         run: |
           cd test-dir/k3s-offline-package
           bash install-server.sh
diff --git a/scripts/make_k3s_offline_package.sh b/scripts/make_k3s_offline_package.sh
index 35e1222..079264b 100644
--- a/scripts/make_k3s_offline_package.sh
+++ b/scripts/make_k3s_offline_package.sh
@@ -285,7 +285,11 @@ rm -rf "${BASE_DIR}"
 mkdir -p "${BASE_DIR}/"{bin,images,cni-plugins,addons,registry/docker.io,registry/ghcr.io,install}
 
 # 核心二进制
-download "${K3S_URL_BASE}/k3s"                "${BASE_DIR}/bin/k3s-${ARCH}"
+K3S_BIN="k3s"
+if [[ "${ARCH}" != "amd64" ]]; then
+  K3S_BIN="k3s-${ARCH}"
+fi
+download "${K3S_URL_BASE}/${K3S_BIN}"                "${BASE_DIR}/bin/k3s-${ARCH}"
 chmod +x "${BASE_DIR}/bin/k3s-${ARCH}"
 
 download "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" "${BASE_DIR}/bin/kubectl-${ARCH}"

From 79c63515a3bb5ce7770db5aec804a03104db2081 Mon Sep 17 00:00:00 2001
From: shenlan <manbuzhe2009@qq.com>
Date: Sun, 14 Sep 2025 17:03:52 +0800
Subject: [PATCH 2/3] Update ingress-installer.sh

Signed-off-by: shenlan <manbuzhe2009@qq.com>
---
 scripts/ingress-installer.sh | 177 +++++++++++++++++++++++++----------
 1 file changed, 130 insertions(+), 47 deletions(-)

diff --git a/scripts/ingress-installer.sh b/scripts/ingress-installer.sh
index e5131b1..d7417fb 100644
--- a/scripts/ingress-installer.sh
+++ b/scripts/ingress-installer.sh
@@ -1,23 +1,23 @@
 #!/usr/bin/env bash
-# gitops/scripts/ingress-installer.sh
-# 目标：最小化参数/分支，专注“一键离线安装”
+# scripts/ingress-installer.sh
+# 目标：一键“离线安装” NGINX Ingress，兼容 K3s 1.29~1.33（containerd）
 set -euo pipefail
 
 # ======================
 # Config & Defaults（仅支持环境变量覆盖）
 # ======================
-: "${NGINX_IC_IMAGE:=nginx/nginx-ingress:2.4.0}"
+: "${NGINX_IC_IMAGE:=nginx/nginx-ingress:5.1.1}"
 : "${CERT_IMG:=registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20230407}"
 
 # 打包阶段写入的 OCI layout 内部引用名（如果你改了打包 ref.name，这里相应改）
-: "${OCI_NGINX_REF:=nginx-ingress-2.4.0}"
+: "${OCI_NGINX_REF:=nginx-ingress-5.1.1}"
 : "${OCI_CERT_REF:=kube-webhook-certgen}"
 
 # 目录布局固定：charts、images、脚本位于离线包根目录
 : "${CHART_DIR:=./charts/nginx-ingress}"
 : "${NAMESPACE:=ingress}"
-: "${OCI_ARCHIVE:=images/oci-archive.tar}"         # 优先使用
-: "${DOCKER_IMG_TAR:=images/nginx-ingress.tar}"    # 回退（docker save）
+: "${OCI_ARCHIVE:=images/oci-archive.tar}"         # 优先使用（oci-archive）
+: "${DOCKER_IMG_TAR:=images/nginx-ingress.tar}"    # 回退（docker-archive）
 : "${DOCKER_CERT_TAR:=images/kube-webhook-certgen.tar}"
 : "${NERDCTL_TAR:=nerdctl.tar.gz}"
 
@@ -33,6 +33,57 @@ ok()   { echo "✅ $*"; }
 warn() { echo "⚠️  $*"; }
 have() { command -v "$1" &>/dev/null; }
 
+# ======================
+# Runtime Detection（K3s / containerd / docker / nerdctl）
+# ======================
+NERDCTL_BIN=""
+detect_nerdctl() {
+  # 优先使用离线包内置路径（install_nerdctl 会解到 /usr/local/bin/nerdctl）
+  if [ -x /usr/local/bin/nerdctl ]; then
+    NERDCTL_BIN="/usr/local/bin/nerdctl"
+  elif have nerdctl; then
+    NERDCTL_BIN="$(command -v nerdctl)"
+  else
+    NERDCTL_BIN=""
+  fi
+}
+have_nerdctl() { [ -n "${NERDCTL_BIN}" ]; }
+
+detect_containerd() {
+  # 优先使用 K3s 的 containerd
+  if [ -S /run/k3s/containerd/containerd.sock ]; then
+    CTR_SOCK="/run/k3s/containerd/containerd.sock"
+    CTR_NS="k8s.io"
+    if have k3s; then
+      CTR_BIN="k3s ctr"  # 避免系统 ctr 指向其他 containerd
+    else
+      CTR_BIN="ctr --address ${CTR_SOCK}"
+    fi
+    NERDCTL_ADDR_OPT=(--address "${CTR_SOCK}" --namespace "${CTR_NS}")
+    return
+  fi
+
+  # 其次使用系统 containerd
+  if [ -S /run/containerd/containerd.sock ]; then
+    CTR_SOCK="/run/containerd/containerd.sock"
+    CTR_NS="k8s.io"
+    CTR_BIN="ctr --address ${CTR_SOCK}"
+    NERDCTL_ADDR_OPT=(--address "${CTR_SOCK}" --namespace "${CTR_NS}")
+    return
+  fi
+
+  CTR_SOCK=""
+}
+
+# 统一的 ctr/nerdctl 执行器
+ctr_exec() {
+  # shellcheck disable=SC2086
+  ${CTR_BIN} -n "${CTR_NS}" "$@"
+}
+nerdctl_exec() {
+  "${NERDCTL_BIN}" "${NERDCTL_ADDR_OPT[@]}" "$@"
+}
+
 # ======================
 # Nerdctl Install (wrapper)
 # ======================
@@ -42,16 +93,26 @@ install_nerdctl() {
     $SUDO tar xzf "${NERDCTL_TAR}" -C /usr/local/bin/
     $SUDO chmod +x /usr/local/bin/nerdctl || true
   fi
+  detect_nerdctl
+  if have_nerdctl; then
+    log "🧰 nerdctl 就绪：${NERDCTL_BIN}"
+  else
+    warn "未检测到 nerdctl，将仅依赖 ctr/docker 执行导入。"
+  fi
 }
 
 # ======================
-# Import images（优先 OCI，其次 docker save tar）
+# Import images（优先 OCI，其次 docker-archive）
 # ======================
 import_images_from_oci() {
-  # 首选：OCI 归档
+  detect_containerd
+  log "🔌 containerd socket: ${CTR_SOCK:-<not found>}"
+
+  # 首选：OCI 归档（oci-archive）—— 标准且最稳
   if [ -f "${OCI_ARCHIVE}" ]; then
     log "📦 从 OCI 归档导入镜像：${OCI_ARCHIVE}"
-    # A) Docker 环境（需要 skopeo）
+
+    # A) Docker 守护进程（需要 skopeo）—— 可选路径
     if have docker && docker info &>/dev/null; then
       if have skopeo; then
         skopeo --insecure-policy copy --all "oci-archive:${OCI_ARCHIVE}:${OCI_NGINX_REF}" "docker-daemon:${NGINX_IC_IMAGE}"
@@ -59,46 +120,68 @@ import_images_from_oci() {
         ok "OCI → docker-daemon 导入完成"
         return
       else
-        warn "docker 环境未安装 skopeo，改用 docker-archive 回退（需 images/*.tar）"
+        warn "docker 环境未安装 skopeo，改用 containerd 路径。"
       fi
     fi
-    # B) containerd 环境（K3s 或系统 containerd）
-    if [ -S /run/k3s/containerd/containerd.sock ] || [ -S /run/containerd/containerd.sock ]; then
-      local ns="k8s.io"
-      $SUDO ctr -n "${ns}" images import --all-platforms "${OCI_ARCHIVE}"
-      $SUDO ctr -n "${ns}" images tag "${OCI_NGINX_REF}" "${NGINX_IC_IMAGE}" || true
-      $SUDO ctr -n "${ns}" images tag "${OCI_CERT_REF}"  "${CERT_IMG}"       || true
+
+    # B) containerd 环境（K3s 或系统 containerd）—— 主路径
+    if [ -n "${CTR_SOCK}" ]; then
+      ctr_exec images import --all-platforms "${OCI_ARCHIVE}"
+      # 补打期望 tag（让 ctr/nerdctl/k8s 三方都一致）
+      ctr_exec images tag "${OCI_NGINX_REF}" "${NGINX_IC_IMAGE}" || true
+      ctr_exec images tag "${OCI_CERT_REF}"  "${CERT_IMG}"       || true
+
+      # 若 nerdctl 可用，再用 nerdctl 做一次 tag（有助于命令行一致性）
+      if have_nerdctl; then
+        nerdctl_exec tag "${OCI_NGINX_REF}" "${NGINX_IC_IMAGE}" || true
+        nerdctl_exec tag "${OCI_CERT_REF}"  "${CERT_IMG}"       || true
+      fi
+
       ok "OCI → containerd 导入完成"
       return
     fi
-    warn "未检测到 docker/skopo 或 containerd 可直接用 OCI 导入，尝试 docker-archive 回退。"
+
+    warn "未检测到可用于 OCI 导入的 containerd，将尝试 docker-archive 回退。"
   fi
 
-  # 回退：docker save 的 tar 包
+  # 回退：docker save 的 tar 包（docker-archive）
   if [ -f "${DOCKER_IMG_TAR}" ] && [ -f "${DOCKER_CERT_TAR}" ]; then
     log "📦 从 docker-archive tar 回退导入 images/*.tar"
+
+    # 优先：nerdctl（离线包自带/系统均可）
+    if have_nerdctl; then
+      if [ -n "${CTR_SOCK}" ]; then
+        nerdctl_exec load -i "${DOCKER_IMG_TAR}"
+        nerdctl_exec load -i "${DOCKER_CERT_TAR}"
+      else
+        # 极少见：未探测到 socket，尝试 nerdctl 默认
+        "${NERDCTL_BIN}" load -i "${DOCKER_IMG_TAR}"
+        "${NERDCTL_BIN}" load -i "${DOCKER_CERT_TAR}"
+      fi
+      ok "nerdctl load 完成"
+      return
+    fi
+
+    # 其次：docker 守护进程
     if have docker && docker info &>/dev/null; then
       docker load -i "${DOCKER_IMG_TAR}"
       docker load -i "${DOCKER_CERT_TAR}"
       ok "docker load 完成"
       return
     fi
-    if have nerdctl; then
-      nerdctl load -i "${DOCKER_IMG_TAR}"
-      nerdctl load -i "${DOCKER_CERT_TAR}"
-      ok "nerdctl load 完成"
-      return
-    fi
-    if [ -S /run/k3s/containerd/containerd.sock ] || [ -S /run/containerd/containerd.sock ]; then
-      $SUDO ctr -n k8s.io images import --all-platforms "${DOCKER_IMG_TAR}"
-      $SUDO ctr -n k8s.io images import --all-platforms "${DOCKER_CERT_TAR}"
+
+    # 兜底：ctr（可导入 docker-archive，但不如 nerdctl 稳妥）
+    if [ -n "${CTR_SOCK}" ]; then
+      ctr_exec images import --all-platforms "${DOCKER_IMG_TAR}"
+      ctr_exec images import --all-platforms "${DOCKER_CERT_TAR}"
       ok "ctr import 完成"
       return
     fi
+
     die "找不到可用容器运行时导入 images/*.tar"
   fi
 
-  die "未发现可用的镜像来源（缺少 ${OCI_ARCHIVE} 或 ${DOCKER_IMG_TAR}/${DOCKER_CERT_TAR)})"
+  die "未发现可用的镜像来源（缺少 ${OCI_ARCHIVE} 或 ${DOCKER_IMG_TAR}/${DOCKER_CERT_TAR}）"
 }
 
 # ======================
@@ -116,29 +199,29 @@ generate_values() {
   tag="${NGINX_IC_IMAGE##*:}"
 
   cat > values.yaml <<EOF
-  controller:
-    ingressClass:
-      name: nginx
-      create: true
-      setAsDefaultIngress: false
-    replicaCount: 2
-  image:
-    repository: ${repo}
-    tag: "${tag}"
-  service:
-    enabled: true
-    type: NodePort
-    externalIPs:
-      - ${INGRESS_IP}
-    nodePorts:
-      http: 80
-      https: 443
+controller:
+  ingressClass:
+    name: nginx
+    create: true
+    setAsDefaultIngress: false
+  replicaCount: 2
+image:
+  repository: ${repo}
+  tag: "${tag}"
+service:
+  enabled: true
+  type: NodePort
+  externalIPs:
+    - ${INGRESS_IP}
+  nodePorts:
+    http: 80
+    https: 443
 EOF
 
   if [[ -n "${NODE_LABEL}" ]]; then
     cat >> values.yaml <<EOF
-  nodeSelector:
-    ${NODE_LABEL%%=*}: "${NODE_LABEL#*=}"
+nodeSelector:
+  ${NODE_LABEL%%=*}: "${NODE_LABEL#*=}"
 EOF
   fi
 }

From 6a92aeb9e231365da20a4693495ca1b6d0bd5ff0 Mon Sep 17 00:00:00 2001
From: shenlan <manbuzhe2009@qq.com>
Date: Sun, 14 Sep 2025 17:25:39 +0800
Subject: [PATCH 3/3] Ensure k3s offline package images match architecture

---
 .../offline-package-k3s-installer.yaml        | 35 +++++++++++-
 scripts/make_k3s_offline_package.sh           | 56 +++++++++++++++++--
 2 files changed, 86 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/offline-package-k3s-installer.yaml b/.github/workflows/offline-package-k3s-installer.yaml
index 7f7f93f..e1f5227 100644
--- a/.github/workflows/offline-package-k3s-installer.yaml
+++ b/.github/workflows/offline-package-k3s-installer.yaml
@@ -76,7 +76,7 @@ jobs:
     needs: build-k3s-installer
     strategy:
       matrix:
-        arch: [amd64]
+        arch: [amd64, arm64]
     runs-on: ubuntu-latest
 
     steps:
@@ -91,7 +91,40 @@ jobs:
           cd test-dir
           tar -xzvf offline-package-k3s-installer-${{ matrix.arch }}.tar.gz
 
+      - name: Install jq
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y jq
+
+      - name: Verify k3s binary arch
+        run: |
+          set -euo pipefail
+          BIN="test-dir/k3s-offline-package/bin/k3s-${{ matrix.arch }}"
+          file "$BIN"
+          if [ "${{ matrix.arch }}" = "amd64" ]; then
+            file "$BIN" | grep -q 'x86-64'
+          else
+            file "$BIN" | grep -q 'aarch64'
+          fi
+
+      - name: Verify image platforms
+        run: |
+          set -euo pipefail
+          TAR="test-dir/k3s-offline-package/images/k3s-airgap-images-${{ matrix.arch }}.tar"
+          tmp=$(mktemp -d)
+          tar -xf "$TAR" -C "$tmp" manifest.json
+          configs=$(jq -r '.[].Config' "$tmp/manifest.json")
+          for cfg in $configs; do
+            tar -xf "$TAR" -C "$tmp" "$cfg"
+            arch=$(jq -r '.architecture' "$tmp/$cfg")
+            if [ "$arch" != "${{ matrix.arch }}" ]; then
+              echo "unexpected arch $arch in $cfg" >&2
+              exit 1
+            fi
+          done
+
       - name: Setup K3s and Test
+        if: matrix.arch == 'amd64'
         run: |
           cd test-dir/k3s-offline-package
           bash install-server.sh
diff --git a/scripts/make_k3s_offline_package.sh b/scripts/make_k3s_offline_package.sh
index 35e1222..6c1894c 100644
--- a/scripts/make_k3s_offline_package.sh
+++ b/scripts/make_k3s_offline_package.sh
@@ -59,14 +59,14 @@ pull_and_save_images() {
   log "拉取核心镜像（runtime=$rt）…"
   case "$rt" in
     docker)
-      for i in "${imgs[@]}"; do docker pull "$i"; done
+      for i in "${imgs[@]}"; do docker pull --platform=linux/${ARCH} "$i"; done
       log "保存镜像 → $out_tar"
       docker save -o "$out_tar" "${imgs[@]}"
       ;;
     nerdctl-default)
-      for i in "${imgs[@]}"; do sudo nerdctl --address /run/containerd/containerd.sock pull "$i"; done
+      for i in "${imgs[@]}"; do sudo nerdctl --address /run/containerd/containerd.sock --platform=linux/${ARCH} pull "$i"; done
       log "保存镜像 → $out_tar"
-      sudo nerdctl --address /run/containerd/containerd.sock save -o "$out_tar" "${imgs[@]}"
+      sudo nerdctl --address /run/containerd/containerd.sock --platform=linux/${ARCH} save -o "$out_tar" "${imgs[@]}"
       ;;
   esac
   [[ -s "$out_tar" ]] || err "未生成镜像包：$out_tar"
@@ -200,6 +200,36 @@ case "$ARCH" in x86_64|amd64) ARCH=amd64;; aarch64|arm64) ARCH=arm64;; *) echo "
 
 BIN_DIR="./bin"
 install_bin(){ sudo cp "$1" "$2"; sudo chmod +x "$2"; echo " ↳ $2"; }
+check_images(){
+  echo "[INFO] 验证已加载镜像架构"
+  local out
+  out=$(sudo nerdctl --namespace k8s.io --address /run/k3s/containerd/containerd.sock images -a --format '{{.Repository}}:{{.Tag}} {{.ID}} {{.Platform}}')
+  echo "$out"
+  if echo "$out" | awk '{print $3}' | grep -v "linux/${ARCH}" >/dev/null; then
+    echo "[ERROR] 发现非 ${ARCH} 架构镜像" >&2
+    exit 1
+  fi
+}
+check_images(){
+  echo "[INFO] 验证已加载镜像架构"
+  local out
+  out=$(sudo nerdctl --namespace k8s.io --address /run/k3s/containerd/containerd.sock images -a --format '{{.Repository}}:{{.Tag}} {{.ID}} {{.Platform}}')
+  echo "$out"
+  if echo "$out" | awk '{print $3}' | grep -v "linux/${ARCH}" >/dev/null; then
+    echo "[ERROR] 发现非 ${ARCH} 架构镜像" >&2
+    exit 1
+  fi
+}
+check_images(){
+  echo "[INFO] 验证已加载镜像架构"
+  local out
+  out=$(sudo nerdctl --namespace k8s.io --address /run/k3s/containerd/containerd.sock images -a --format '{{.Repository}}:{{.Tag}} {{.ID}} {{.Platform}}')
+  echo "$out"
+  if echo "$out" | awk '{print $3}' | grep -v "linux/${ARCH}" >/dev/null; then
+    echo "[ERROR] 发现非 ${ARCH} 架构镜像" >&2
+    exit 1
+  fi
+}
 
 echo "[INFO] 安装 CLI → /usr/local/bin"
 install_bin "${BIN_DIR}/k3s-${ARCH}" /usr/local/bin/k3s
@@ -220,6 +250,8 @@ mkdir -p ~/.kube && cp -f /etc/rancher/k3s/k3s.yaml ~/.kube/config || true
 kubectl apply -f addons/node-exporter.yaml || true
 kubectl apply -f addons/kube-state-metrics.yaml || true
 
+check_images
+
 echo "[SUCCESS] 离线 K3s 安装完成 ✅"
 SH
   chmod +x "${BASE_DIR}/install-server.sh"
@@ -235,6 +267,16 @@ case "$ARCH" in x86_64|amd64) ARCH=amd64;; aarch64|arm64) ARCH=arm64;; *) echo "
 
 BIN_DIR="./bin"
 install_bin(){ sudo cp "$1" "$2"; sudo chmod +x "$2"; echo " ↳ $2"; }
+check_images(){
+  echo "[INFO] 验证已加载镜像架构"
+  local out
+  out=$(sudo nerdctl --namespace k8s.io --address /run/k3s/containerd/containerd.sock images -a --format '{{.Repository}}:{{.Tag}} {{.ID}} {{.Platform}}')
+  echo "$out"
+  if echo "$out" | awk '{print $3}' | grep -v "linux/${ARCH}" >/dev/null; then
+    echo "[ERROR] 发现非 ${ARCH} 架构镜像" >&2
+    exit 1
+  fi
+}
 
 echo "[INFO] 安装 CLI → /usr/local/bin"
 install_bin "${BIN_DIR}/k3s-${ARCH}" /usr/local/bin/k3s
@@ -246,6 +288,8 @@ INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC="agent" bash install/k3s-officia
 echo "[INFO] 加载 airgap 镜像"
 sudo nerdctl --namespace k8s.io --address /run/k3s/containerd/containerd.sock load -i "images/k3s-airgap-images-${ARCH}.tar" || true
 
+check_images
+
 echo "[SUCCESS] Agent 节点离线安装完成 ✅"
 SH
   chmod +x "${BASE_DIR}/install-agent.sh"
@@ -285,7 +329,11 @@ rm -rf "${BASE_DIR}"
 mkdir -p "${BASE_DIR}/"{bin,images,cni-plugins,addons,registry/docker.io,registry/ghcr.io,install}
 
 # 核心二进制
-download "${K3S_URL_BASE}/k3s"                "${BASE_DIR}/bin/k3s-${ARCH}"
+K3S_BIN="k3s"
+if [[ "${ARCH}" != "amd64" ]]; then
+  K3S_BIN="k3s-${ARCH}"
+fi
+download "${K3S_URL_BASE}/${K3S_BIN}"                "${BASE_DIR}/bin/k3s-${ARCH}"
 chmod +x "${BASE_DIR}/bin/k3s-${ARCH}"
 
 download "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" "${BASE_DIR}/bin/kubectl-${ARCH}"