name: Build Offline AI Workspace All-in-One Package

on:
  workflow_dispatch:
    inputs:
      tag:
        description: "Release tag. Leave empty to use offline-ai-workspace-<run_number>"
        required: false
        type: string
      playbooks_ref:
        description: "ai-workspace-infra/playbooks git ref"
        required: false
        default: "main"
        type: string
      console_ref:
        description: "ai-workspace-lab/xworkspace-console git ref"
        required: false
        default: "main"
        type: string
      core_skills_ref:
        description: "ai-workspace-lab/xworkspace-core-skills git ref"
        required: false
        default: "main"
        type: string
      console_runtime_release_tag:
        description: "Console runtime release tag, or latest-runtime"
        required: false
        default: "latest-runtime"
        type: string
      bridge_runtime_release_tag:
        description: "Bridge runtime release tag, or latest-runtime"
        required: false
        default: "latest-runtime"
        type: string
      qmd_runtime_release_tag:
        description: "QMD runtime release tag, or latest-runtime"
        required: false
        default: "latest-runtime"
        type: string
      litellm_runtime_release_tag:
        description: "LiteLLM runtime release tag, or latest-runtime"
        required: false
        default: "latest-runtime"
        type: string

permissions:
  contents: write

concurrency:
  group: build-offline-ai-workspace-all-in-one
  cancel-in-progress: false

jobs:
  build-offline-package:
    name: Build ${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - distro: debian
            version: "13"
            arch: amd64
          - distro: debian
            version: "13"
            arch: arm64
          - distro: debian
            version: "12"
            arch: amd64
          - distro: debian
            version: "12"
            arch: arm64
          - distro: debian
            version: "11"
            arch: amd64
          - distro: debian
            version: "11"
            arch: arm64
          - distro: ubuntu
            version: "26.04"
            arch: amd64
          - distro: ubuntu
            version: "26.04"
            arch: arm64
          - distro: ubuntu
            version: "24.04"
            arch: amd64
          - distro: ubuntu
            version: "24.04"
            arch: arm64
          - distro: ubuntu
            version: "22.04"
            arch: amd64
          - distro: ubuntu
            version: "22.04"
            arch: arm64
    steps:
      - uses: actions/checkout@v4

      - name: Enable QEMU for cross-arch package collection
        uses: docker/setup-qemu-action@v3

      - name: Install build dependencies
        run: |
          set -euo pipefail
          sudo apt-get update -y
          sudo apt-get install -y curl git jq python3-pip

      - name: Build offline package
        env:
          DISTRO_ID: ${{ matrix.distro }}
          DISTRO_VERSION: ${{ matrix.version }}
          ARCH: ${{ matrix.arch }}
          PLAYBOOKS_REF: ${{ github.event.inputs.playbooks_ref || 'main' }}
          CONSOLE_REF: ${{ github.event_name == 'push' && github.sha || github.event.inputs.console_ref || 'main' }}
          CORE_SKILLS_REF: ${{ github.event.inputs.core_skills_ref || 'main' }}
          CONSOLE_RUNTIME_RELEASE_TAG: ${{ github.event.inputs.console_runtime_release_tag || 'latest-runtime' }}
          BRIDGE_RUNTIME_RELEASE_TAG: ${{ github.event.inputs.bridge_runtime_release_tag || 'latest-runtime' }}
          QMD_RUNTIME_RELEASE_TAG: ${{ github.event.inputs.qmd_runtime_release_tag || 'latest-runtime' }}
          LITELLM_RUNTIME_RELEASE_TAG: ${{ github.event.inputs.litellm_runtime_release_tag || 'latest-runtime' }}
          GH_TOKEN: ${{ github.token }}
          PACKAGE_VERSION: ${{ github.run_number }}
        run: |
          set -euo pipefail
          chmod +x scripts/create-ai-workspace-offline-package.sh
          scripts/create-ai-workspace-offline-package.sh

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: ai-workspace-all-in-one-offline-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}
          path: ai-workspace-all-in-one-offline-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}.tar.gz
          if-no-files-found: error

  test-offline-package:
    needs: build-offline-package
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - distro: debian
            version: "13"
            arch: amd64
          - distro: ubuntu
            version: "24.04"
            arch: amd64
          - distro: debian
            version: "13"
            arch: arm64
          - distro: ubuntu
            version: "24.04"
            arch: arm64
    steps:
      - uses: actions/checkout@v4

      - name: Verify online bootstrap hands off to offline installer
        run: |
          set -euo pipefail
          fixture="${RUNNER_TEMP}/offline-handoff"
          mkdir -p "${fixture}/scripts" "${fixture}/metadata"
          cat > "${fixture}/metadata/target.env" <<EOF
          DISTRO_ID=${{ matrix.distro }}
          DISTRO_VERSION=${{ matrix.version }}
          ARCH=${{ matrix.arch }}
          EOF
          cat > "${fixture}/scripts/ai-workspace-offline-install.sh" <<'EOF'
          #!/usr/bin/env bash
          set -euo pipefail
          test "${AI_WORKSPACE_OFFLINE_ACTIVE:-}" = "true"
          printf 'offline-handoff-ok\n'
          EOF
          chmod +x "${fixture}/scripts/ai-workspace-offline-install.sh"
          output="$(
            AI_WORKSPACE_OFFLINE_MODE=force \
            AI_WORKSPACE_OFFLINE_PACKAGE="${fixture}" \
            AI_WORKSPACE_OFFLINE_DISTRO_ID=${{ matrix.distro }} \
            AI_WORKSPACE_OFFLINE_DISTRO_VERSION=${{ matrix.version }} \
            AI_WORKSPACE_OFFLINE_ARCH=${{ matrix.arch }} \
            bash scripts/setup-ai-workspace-all-in-one.sh
          )"
          grep -q 'offline-handoff-ok' <<<"${output}"

      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: ai-workspace-all-in-one-offline-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}
          path: offline-test

      - name: Verify offline package contents
        run: |
          set -euo pipefail
          cd offline-test
          tar -tzf ai-workspace-all-in-one-offline-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}.tar.gz > contents.txt
          grep -q 'scripts/ai-workspace-offline-install.sh' contents.txt
          grep -q 'metadata/manifest.json' contents.txt
          grep -q 'metadata/target.env' contents.txt
          grep -q 'repos/playbooks' contents.txt
          grep -q 'repos/xworkspace-console' contents.txt
          grep -q 'packages/components/xworkspace-console-runtime-linux-${{ matrix.arch }}.tar.gz' contents.txt
          grep -q 'packages/components/xworkmate-bridge-linux-${{ matrix.arch }}.tar.gz' contents.txt
          grep -q 'packages/components/qmd-runtime-linux-${{ matrix.arch }}.tar.gz' contents.txt
          grep -q 'packages/components/litellm-runtime-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}.tar.gz' contents.txt
          grep -q 'metadata/litellm-runtime.env' contents.txt
          grep -q 'metadata/components/xworkmate-bridge.tag' contents.txt
          grep -q 'packages/apt/Packages.gz' contents.txt
          if [[ "${{ matrix.arch }}" == "arm64" ]]; then
            grep -Eq 'packages/playwright-browsers/.*/chrome-linux(64)?/chrome$' contents.txt
            grep -q 'metadata/apt/browser-deb-packages.txt' contents.txt
          fi
          if [[ "${{ matrix.distro }}:${{ matrix.version }}" == "ubuntu:26.04" ]]; then
            grep -Eq 'packages/python/.*/bin/python3.13$' contents.txt
          fi

  publish-release:
    needs: test-offline-package
    runs-on: ubuntu-latest
    env:
      TAG_NAME: ${{ github.event.inputs.tag != '' && github.event.inputs.tag || format('offline-ai-workspace-{0}', github.run_number) }}
      CONSOLE_RUNTIME_RELEASE_TAG: ${{ github.event.inputs.console_runtime_release_tag || 'latest-runtime' }}
      RSYNC_SSH_KEY: ${{ secrets.RSYNC_SSH_KEY }}
      RSYNC_SSH_USER: ${{ secrets.RSYNC_SSH_USER }}
      VPS_HOST: ${{ secrets.VPS_HOST }}
      REMOTE_ROOT: /data/update-server/offline-package/ai-workspace
    steps:
      - uses: actions/checkout@v4

      - name: Create GitHub Release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          if gh release view "$TAG_NAME" >/dev/null 2>&1; then
            echo "Release $TAG_NAME already exists"
          else
            gh release create "$TAG_NAME" --title "Build $TAG_NAME" --notes "Offline AI Workspace all-in-one packages."
          fi

      - name: Download all package artifacts
        uses: actions/download-artifact@v4
        with:
          path: release-artifacts
          pattern: ai-workspace-all-in-one-offline-*
          merge-multiple: true

      - name: Download console runtime release assets
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          mkdir -p release-artifacts
          runtime_tag="$CONSOLE_RUNTIME_RELEASE_TAG"
          if [[ "$runtime_tag" == "latest-runtime" || -z "$runtime_tag" ]]; then
            runtime_tag="$(
              gh release list \
                --limit 50 \
                --json tagName \
                --jq '[.[].tagName | select(startswith("runtime-"))][0]'
            )"
          fi
          if [[ -z "$runtime_tag" || "$runtime_tag" == "null" ]]; then
            echo "Unable to resolve console runtime release tag" >&2
            exit 1
          fi
          echo "Downloading console runtime assets from ${runtime_tag}"
          gh release download "$runtime_tag" \
            --pattern 'xworkspace-console-runtime-*.tar.gz' \
            --pattern 'SHA256SUMS' \
            --dir release-artifacts \
            --clobber

      - name: Upload packages to GitHub Release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          shopt -s nullglob

          # GitHub release assets are hard-capped at 2 GiB. Packages at/over the
          # limit are split into <2 GiB parts plus a ".parts" manifest; the
          # offline bootstrap reassembles them. Parts are 1900 MiB.
          GH_ASSET_LIMIT=2147483648
          SPLIT_SIZE=1900m

          delete_asset_if_present() {
            local name=$1
            if gh release view "$TAG_NAME" --json assets --jq '.assets[].name' \
                 2>/dev/null | grep -Fxq "$name"; then
              echo "Deleting existing release asset ${name}"
              gh release delete-asset "$TAG_NAME" "$name" --yes
            fi
          }

          upload_one() {
            local file=$1 name attempt
            name="$(basename "$file")"
            delete_asset_if_present "$name"
            for attempt in 1 2 3; do
              if gh release upload "$TAG_NAME" "$file" --clobber; then
                return 0
              fi
              if [[ "$attempt" -eq 3 ]]; then
                echo "Failed to upload ${file} after ${attempt} attempts" >&2
                return 1
              fi
              sleep $((attempt * 20))
            done
          }

          packages=(release-artifacts/*.tar.gz)
          if [[ ${#packages[@]} -eq 0 ]]; then
            echo "No offline packages found in release-artifacts" >&2
            exit 1
          fi

          for package in "${packages[@]}"; do
            asset_name="$(basename "$package")"
            size="$(stat -c%s "$package")"
            if [[ "$size" -lt "$GH_ASSET_LIMIT" ]]; then
              echo "Uploading ${asset_name} (${size} bytes)"
              upload_one "$package"
            else
              echo "Splitting oversized ${asset_name} (${size} bytes) into ${SPLIT_SIZE} parts"
              dir="$(dirname "$package")"
              (
                cd "$dir"
                rm -f "${asset_name}".part-* "${asset_name}.parts"
                split -b "$SPLIT_SIZE" -d -a 3 "$asset_name" "${asset_name}.part-"
                ls "${asset_name}".part-* | LC_ALL=C sort > "${asset_name}.parts"
              )
              # Remove any stale whole asset, then publish the manifest + parts.
              delete_asset_if_present "$asset_name"
              upload_one "${dir}/${asset_name}.parts"
              for part in "${dir}/${asset_name}".part-*; do
                echo "Uploading $(basename "$part")"
                upload_one "$part"
              done
            fi
          done

      - name: Rsync packages to remote mirror
        if: ${{ env.RSYNC_SSH_KEY != '' && env.RSYNC_SSH_USER != '' && env.VPS_HOST != '' }}
        run: |
          set -euo pipefail
          sudo apt-get update -y
          sudo apt-get install -y rsync openssh-client
          mkdir -p ~/.ssh
          echo "$RSYNC_SSH_KEY" > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
          ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts
          remote_dir="${REMOTE_ROOT}/${TAG_NAME}"
          ssh -i ~/.ssh/id_rsa "${RSYNC_SSH_USER}@${VPS_HOST}" "mkdir -p '${remote_dir}'"
          rsync -av -e "ssh -i ~/.ssh/id_rsa" release-artifacts/*.tar.gz \
            "${RSYNC_SSH_USER}@${VPS_HOST}:${remote_dir}/"
