diff --git a/.github/workflows/offline-package-ai-workspace-installer.yaml b/.github/workflows/offline-package-ai-workspace-installer.yaml index 47e6cf9..a95702f 100644 --- a/.github/workflows/offline-package-ai-workspace-installer.yaml +++ b/.github/workflows/offline-package-ai-workspace-installer.yaml @@ -128,6 +128,12 @@ jobs: - distro: ubuntu version: "24.04" arch: amd64 + - distro: debian + version: "13" + arch: arm64 + - distro: ubuntu + version: "24.04" + arch: arm64 steps: - uses: actions/checkout@v4 @@ -175,6 +181,10 @@ jobs: grep -q 'repos/playbooks' contents.txt grep -q 'repos/xworkspace-console' contents.txt grep -q 'packages/apt/Packages.gz' contents.txt + if [[ "${{ matrix.arch }}" == "arm64" ]]; then + grep -Eq 'packages/playwright-browsers/.*/chrome-linux(64)?/chrome$' contents.txt + grep -q 'metadata/apt/browser-deb-packages.txt' contents.txt + fi publish-release: needs: test-offline-package diff --git a/scripts/ai-workspace-offline-install.sh b/scripts/ai-workspace-offline-install.sh index 2c176bd..de39370 100755 --- a/scripts/ai-workspace-offline-install.sh +++ b/scripts/ai-workspace-offline-install.sh @@ -8,10 +8,14 @@ IMAGE_DIR="${ROOT}/packages/images" NPM_CACHE_DIR="${ROOT}/packages/npm-cache" NPM_RUNTIME_CACHE_DIR="${AI_WORKSPACE_NPM_CACHE_DIR:-/var/cache/ai-workspace/npm}" PIP_WHEEL_DIR="${ROOT}/packages/pip" +PLAYWRIGHT_BROWSER_DIR="${ROOT}/packages/playwright-browsers" +PLAYWRIGHT_BROWSER_INSTALL_DIR="${AI_WORKSPACE_PLAYWRIGHT_BROWSER_DIR:-/opt/ai-workspace/playwright-browsers}" STATE_DIR="${AI_WORKSPACE_OFFLINE_STATE_DIR:-/var/lib/ai-workspace/offline}" AI_WORKSPACE_DEPLOYMENT_LOCK_TIMEOUT="${AI_WORKSPACE_DEPLOYMENT_LOCK_TIMEOUT:-1800}" AI_WORKSPACE_APT_LOCK_TIMEOUT="${AI_WORKSPACE_APT_LOCK_TIMEOUT:-900}" APT_SOURCE_FILE="/etc/apt/sources.list.d/ai-workspace-offline.list" +APT_CONFIG_FILE="/etc/apt/apt.conf.d/99ai-workspace-offline" +BROWSER_APT_PACKAGE_FILE="${ROOT}/metadata/apt/browser-deb-packages.txt" SAFE_GIT_DIRS=() APT_LOCAL_OPTIONS=( -o "Dir::Etc::sourcelist=sources.list.d/ai-workspace-offline.list" @@ -28,7 +32,7 @@ warn() { } cleanup() { - rm -f "${APT_SOURCE_FILE}" + rm -f "${APT_SOURCE_FILE}" "${APT_CONFIG_FILE}" local git_dir for git_dir in "${SAFE_GIT_DIRS[@]}"; do git config --system --unset-all safe.directory "${git_dir}" >/dev/null 2>&1 || true @@ -117,6 +121,12 @@ configure_local_apt_repo() { info "Configuring local APT repository from ${APT_DIR}" cat > "${APT_SOURCE_FILE}" < "${APT_CONFIG_FILE}" <<'EOF' +Dir::Etc::sourcelist "sources.list.d/ai-workspace-offline.list"; +Dir::Etc::sourceparts "-"; +APT::Get::List-Cleanup "0"; +Acquire::Languages "none"; EOF apt-get "${APT_LOCAL_OPTIONS[@]}" update } @@ -146,6 +156,17 @@ append_available_package() { install_offline_prerequisites() { local packages=(git curl ansible ca-certificates unzip) + local browser_package + + if [ -d "${PLAYWRIGHT_BROWSER_DIR}" ] && [ ! -s "${BROWSER_APT_PACKAGE_FILE}" ]; then + echo "Bundled Playwright Chromium dependency manifest is missing." >&2 + exit 1 + fi + if [ -d "${PLAYWRIGHT_BROWSER_DIR}" ]; then + while IFS= read -r browser_package; do + [ -n "${browser_package}" ] && packages+=("${browser_package}") + done < "${BROWSER_APT_PACKAGE_FILE}" + fi if ! command -v docker >/dev/null 2>&1; then if apt-cache "${APT_LOCAL_OPTIONS[@]}" show docker-ce >/dev/null 2>&1; then @@ -182,6 +203,57 @@ install_bundled_binaries() { info "Installing bundled ttyd binary" install -m 0755 "${BIN_DIR}/ttyd.${ttyd_arch}" /usr/local/bin/ttyd fi + + case "$(uname -m)" in + x86_64|amd64) bridge_arch=amd64 ;; + aarch64|arm64) bridge_arch=arm64 ;; + *) bridge_arch="" ;; + esac + if [ -n "${bridge_arch}" ] && [ -f "${BIN_DIR}/xworkmate-go-core.${bridge_arch}" ]; then + info "Installing bundled XWorkmate Bridge binary" + local bridge_immutable=false + if command -v lsattr >/dev/null 2>&1 && + lsattr /usr/local/bin/xworkmate-go-core 2>/dev/null | awk '{print $1}' | grep -q 'i'; then + bridge_immutable=true + chattr -i /usr/local/bin/xworkmate-go-core + fi + install -m 0755 "${BIN_DIR}/xworkmate-go-core.${bridge_arch}" /usr/local/bin/xworkmate-go-core + if [ "${bridge_immutable}" = "true" ]; then + chattr +i /usr/local/bin/xworkmate-go-core + fi + fi +} + +install_bundled_playwright_browser() { + if [ ! -d "${PLAYWRIGHT_BROWSER_DIR}" ]; then + return + fi + + local seed_checksum marker browser_binary + seed_checksum="$(sha256sum "${ROOT}/metadata/manifest.json" | awk '{print $1}')" + marker="${PLAYWRIGHT_BROWSER_INSTALL_DIR}/.ai-workspace-seed-sha256" + if [ "$(cat "${marker}" 2>/dev/null || true)" != "${seed_checksum}" ]; then + info "Installing bundled Playwright Chromium runtime" + rm -rf "${PLAYWRIGHT_BROWSER_INSTALL_DIR}" + mkdir -p "${PLAYWRIGHT_BROWSER_INSTALL_DIR}" + cp -a "${PLAYWRIGHT_BROWSER_DIR}/." "${PLAYWRIGHT_BROWSER_INSTALL_DIR}/" + printf '%s\n' "${seed_checksum}" > "${marker}" + else + info "Reusing bundled Playwright Chromium runtime" + fi + + chown -R root:root "${PLAYWRIGHT_BROWSER_INSTALL_DIR}" + chmod -R a+rX "${PLAYWRIGHT_BROWSER_INSTALL_DIR}" + browser_binary="$( + find "${PLAYWRIGHT_BROWSER_INSTALL_DIR}" -type f \ + \( -path '*/chrome-linux/chrome' -o -path '*/chrome-linux64/chrome' \) \ + -print -quit + )" + if [ -z "${browser_binary}" ] || [ ! -x "${browser_binary}" ]; then + echo "Bundled Playwright Chromium executable is missing." >&2 + exit 1 + fi + ln -sfn "${browser_binary}" /usr/local/bin/chromium } load_container_images() { @@ -282,6 +354,7 @@ run_bootstrap() { export GATEWAY_OPENCLAW_PUBLIC_ACCESS="${GATEWAY_OPENCLAW_PUBLIC_ACCESS:-false}" export VAULT_PUBLIC_ACCESS="${VAULT_PUBLIC_ACCESS:-false}" export AI_WORKSPACE_OFFLINE_ACTIVE=true + export AI_WORKSPACE_USE_PREBUILT_BRIDGE=true export AI_WORKSPACE_DEPLOYMENT_LOCK_HELD=true export AI_WORKSPACE_APT_LOCK_TIMEOUT @@ -298,6 +371,7 @@ main() { install_offline_prerequisites configure_local_git_sources install_bundled_binaries + install_bundled_playwright_browser load_container_images configure_language_package_caches run_bootstrap diff --git a/scripts/create-ai-workspace-offline-package.sh b/scripts/create-ai-workspace-offline-package.sh index 68aeaa2..0cc7878 100755 --- a/scripts/create-ai-workspace-offline-package.sh +++ b/scripts/create-ai-workspace-offline-package.sh @@ -27,6 +27,7 @@ TTYD_VERSION="${TTYD_VERSION:-latest}" POSTGRES_IMAGE="${POSTGRES_IMAGE:-postgres:17.7}" OPENCLAW_VERSION="${OPENCLAW_VERSION:-2026.6.6}" PLAYWRIGHT_VERSION="${PLAYWRIGHT_VERSION:-1.60.0}" +GOOGLE_CHROME_VERSION="${GOOGLE_CHROME_VERSION:-149.0.7827.114-1}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" @@ -86,6 +87,7 @@ download_apt_packages() { -e NODEJS_MAJOR_VERSIONS="${NODEJS_MAJOR_VERSIONS}" \ -e NODEJS_22_VERSION="${NODEJS_22_VERSION}" \ -e NODEJS_24_VERSION="${NODEJS_24_VERSION}" \ + -e GOOGLE_CHROME_VERSION="${GOOGLE_CHROME_VERSION}" \ -v "${apt_dir}:/offline-apt" \ -v "${apt_lists}:/offline-meta" \ "${image}" \ @@ -101,10 +103,12 @@ for major in ${NODEJS_MAJOR_VERSIONS}; do echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${major}.x nodistro main" \ > "/etc/apt/sources.list.d/nodesource-node${major}.list" done -curl -fsSL https://dl.google.com/linux/linux_signing_key.pub \ - | gpg --dearmor -o /etc/apt/keyrings/google-linux-signing-key.gpg -echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/google-linux-signing-key.gpg] https://dl.google.com/linux/chrome/deb/ stable main" \ - > /etc/apt/sources.list.d/google-chrome.list +if [ "$(dpkg --print-architecture)" = "amd64" ]; then + curl -fsSL https://dl.google.com/linux/linux_signing_key.pub \ + | gpg --dearmor -o /etc/apt/keyrings/google-linux-signing-key.gpg + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/google-linux-signing-key.gpg] https://dl.google.com/linux/chrome/deb/ stable main" \ + > /etc/apt/sources.list.d/google-chrome.list +fi curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg -o /etc/apt/keyrings/yarn.asc echo "deb [signed-by=/etc/apt/keyrings/yarn.asc] https://dl.yarnpkg.com/debian/ stable main" \ > /etc/apt/sources.list.d/yarn.list @@ -118,14 +122,66 @@ fi apt-get update -y || true required_packages=( - ansible git curl ca-certificates gnupg jq rsync unzip wget -) -optional_packages=( + ansible git curl ca-certificates gnupg jq rsync unzip wget xdg-utils caddy xfce4 python3 python3-pip python3-venv python3-dev python3-setuptools - build-essential pkg-config python-is-python3 pandoc fonts-noto-cjk + build-essential pkg-config python-is-python3 pandoc nodejs yarn golang-go + texlive-xetex texlive-latex-extra texlive-fonts-recommended + texlive-lang-chinese latexmk fonts-noto-cjk fonts-noto-cjk-extra fonts-wqy-zenhei fonts-wqy-microhei - google-chrome-stable nodejs yarn ) + +resolve_required_package() { + local candidate + for candidate in "$@"; do + if apt-cache show "${candidate}" >/dev/null 2>&1; then + printf '%s\n' "${candidate}" + return + fi + done + echo "Required APT package is unavailable for this target: $*" >&2 + exit 1 +} + +case "$(dpkg --print-architecture)" in + amd64) + if ! apt-cache show "google-chrome-stable=${GOOGLE_CHROME_VERSION}" >/dev/null 2>&1; then + echo "Required Google Chrome version is unavailable: ${GOOGLE_CHROME_VERSION}" >&2 + exit 1 + fi + ;; + arm64) + browser_dependency_groups=( + "libasound2 libasound2t64" + "libatk-bridge2.0-0 libatk-bridge2.0-0t64" + "libatk1.0-0 libatk1.0-0t64" + "libatspi2.0-0 libatspi2.0-0t64" + "libcairo2" + "libcups2 libcups2t64" + "libdbus-1-3" + "libdrm2" + "libgbm1" + "libglib2.0-0 libglib2.0-0t64" + "libnspr4" + "libnss3" + "libpango-1.0-0" + "libx11-6" + "libxcb1" + "libxcomposite1" + "libxdamage1" + "libxext6" + "libxfixes3" + "libxkbcommon0" + "libxrandr2" + ) + : > /offline-meta/browser-deb-packages.txt + for dependency_group in "${browser_dependency_groups[@]}"; do + read -r -a dependency_candidates <<<"${dependency_group}" + resolved_dependency="$(resolve_required_package "${dependency_candidates[@]}")" + required_packages+=("${resolved_dependency}") + printf '%s\n' "${resolved_dependency}" >> /offline-meta/browser-deb-packages.txt + done + ;; +esac packages=() for package in "${required_packages[@]}"; do @@ -136,13 +192,9 @@ for package in "${required_packages[@]}"; do packages+=("${package}") done -for package in "${optional_packages[@]}"; do - if apt-cache show "${package}" >/dev/null 2>&1; then - packages+=("${package}") - else - echo "Skipping unavailable optional APT package: ${package}" >&2 - fi -done +if [ "$(dpkg --print-architecture)" = "amd64" ]; then + packages+=("google-chrome-stable=${GOOGLE_CHROME_VERSION}") +fi if apt-cache show docker-ce >/dev/null 2>&1; then packages+=(docker-ce docker-ce-cli containerd.io) @@ -158,25 +210,16 @@ elif apt-cache show docker.io >/dev/null 2>&1; then fi fi -if apt-cache show golang-go >/dev/null 2>&1; then - packages+=(golang-go) -fi -if apt-cache show texlive-xetex >/dev/null 2>&1; then - packages+=(texlive-xetex texlive-latex-extra texlive-fonts-recommended texlive-lang-chinese latexmk) -fi - apt-get install --download-only -y --no-install-recommends "${packages[@]}" -apt-get download "nodejs=${NODEJS_22_VERSION}-1nodesource1" || true -apt-get download "nodejs=${NODEJS_24_VERSION}-1nodesource1" || true +apt-get download "nodejs=${NODEJS_22_VERSION}-1nodesource1" +apt-get download "nodejs=${NODEJS_24_VERSION}-1nodesource1" cp -n ./*.deb /offline-apt/ 2>/dev/null || true find /var/cache/apt/archives -name '*.deb' -exec cp -n {} /offline-apt/ \; cd /offline-apt -dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz +dpkg-scanpackages --multiversion . /dev/null | gzip -9c > Packages.gz find . -maxdepth 1 -name '*.deb' -printf '%f\n' | sort > /offline-meta/deb-files.txt CONTAINER -)" \ - -e DISTRO_ID="${DISTRO_ID}" \ - -e NODEJS_MAJOR_VERSIONS="${NODEJS_MAJOR_VERSIONS}" +)" } download_npm_packages() { @@ -199,6 +242,68 @@ download_npm_packages() { npm cache add --cache "${npm_cache_dir}" "playwright@${PLAYWRIGHT_VERSION}" } +warm_npm_dependency_cache() { + local npm_cache_dir="${WORKDIR}/packages/npm-cache" + docker run --rm --platform "linux/${ARCH}" \ + -e OPENCLAW_VERSION="${OPENCLAW_VERSION}" \ + -e PLAYWRIGHT_VERSION="${PLAYWRIGHT_VERSION}" \ + -v "${npm_cache_dir}:/offline-cache" \ + -v "${WORKDIR}/repos/xworkspace-console/dashboard:/sources/dashboard:ro" \ + -v "${WORKDIR}/repos/qmd:/sources/qmd:ro" \ + node:24-bookworm \ + bash -lc "$(cat <<'CONTAINER' +set -euo pipefail +export npm_config_cache=/offline-cache +export npm_config_audit=false +export npm_config_fund=false + +mkdir -p /tmp/projects +cp -a /sources/dashboard /tmp/projects/dashboard +cp -a /sources/qmd /tmp/projects/qmd + +( + cd /tmp/projects/dashboard + npm ci --ignore-scripts --no-audit --no-fund +) +( + cd /tmp/projects/qmd + npm install --ignore-scripts --no-audit --no-fund --package-lock=false +) + +npm install --ignore-scripts --no-audit --no-fund --prefix /tmp/global \ + opencode-ai \ + @google/gemini-cli \ + @openai/codex \ + @anthropic-ai/claude-code \ + "openclaw@${OPENCLAW_VERSION}" \ + "@openclaw/codex@${OPENCLAW_VERSION}" \ + "playwright@${PLAYWRIGHT_VERSION}" +CONTAINER +)" +} + +download_playwright_browser() { + if [ "${ARCH}" != "arm64" ]; then + return + fi + + local browser_dir="${WORKDIR}/packages/playwright-browsers" + mkdir -p "${browser_dir}" + docker run --rm --platform "linux/${ARCH}" \ + -e PLAYWRIGHT_VERSION="${PLAYWRIGHT_VERSION}" \ + -e PLAYWRIGHT_BROWSERS_PATH=/offline-browsers \ + -v "${browser_dir}:/offline-browsers" \ + node:24-bookworm \ + bash -lc 'npx --yes "playwright@${PLAYWRIGHT_VERSION}" install chromium' + + if ! find "${browser_dir}" -type f \ + \( -path '*/chrome-linux/chrome' -o -path '*/chrome-linux64/chrome' \) \ + -print -quit | grep -q .; then + echo "Bundled Playwright Chromium executable was not found for ${ARCH}" >&2 + exit 1 + fi +} + download_pip_wheels() { local wheel_dir="${WORKDIR}/packages/pip" mkdir -p "${wheel_dir}" @@ -232,6 +337,20 @@ download_binaries() { chmod +x "${bin_dir}/ttyd.${ttyd_arch}" } +build_xworkmate_bridge_binary() { + local bin_dir="${WORKDIR}/packages/bin" + mkdir -p "${bin_dir}" + docker run --rm --platform "linux/${ARCH}" \ + -e GOOS=linux \ + -e GOARCH="${ARCH}" \ + -e CGO_ENABLED=0 \ + -v "${WORKDIR}/repos/xworkmate-bridge:/src:ro" \ + -v "${bin_dir}:/out" \ + golang:1.25-bookworm \ + bash -lc 'cd /src && go build -trimpath -o "/out/xworkmate-go-core.${GOARCH}" .' + chmod +x "${bin_dir}/xworkmate-go-core.${ARCH}" +} + export_container_images() { local images_dir="${WORKDIR}/packages/images" mkdir -p "${images_dir}" @@ -268,6 +387,7 @@ EOF "vault": "${VAULT_VERSION}", "openclaw": "${OPENCLAW_VERSION}", "playwright": "${PLAYWRIGHT_VERSION}", + "googleChrome": "${GOOGLE_CHROME_VERSION}", "postgresImage": "${POSTGRES_IMAGE}" } } @@ -293,8 +413,11 @@ main() { download_apt_packages "${image}" download_npm_packages + warm_npm_dependency_cache + download_playwright_browser download_pip_wheels download_binaries + build_xworkmate_bridge_binary export_container_images cp "${SCRIPT_DIR}/ai-workspace-offline-install.sh" "${WORKDIR}/scripts/" diff --git a/scripts/setup-ai-workspace-all-in-one.sh b/scripts/setup-ai-workspace-all-in-one.sh index 04cc97a..3ce4f1d 100755 --- a/scripts/setup-ai-workspace-all-in-one.sh +++ b/scripts/setup-ai-workspace-all-in-one.sh @@ -10,6 +10,7 @@ set -euo pipefail # Supported Environment Variables: # AI_WORKSPACE_SECURITY_LEVEL # LITELLM_API_CADDY_STRICT_WHITELIST +# LITELLM_CADDY_CONFIG_ENABLED # XWORKSPACE_CONSOLE_PUBLIC_ACCESS # XWORKMATE_BRIDGE_PUBLIC_ACCESS # GATEWAY_OPENCLAW_PUBLIC_ACCESS @@ -490,6 +491,7 @@ run_offline_installer() { for env_name in \ AI_WORKSPACE_SECURITY_LEVEL \ LITELLM_API_CADDY_STRICT_WHITELIST \ + LITELLM_CADDY_CONFIG_ENABLED \ XWORKSPACE_CONSOLE_PUBLIC_ACCESS \ XWORKMATE_BRIDGE_PUBLIC_ACCESS \ XWORKMATE_BRIDGE_DOMAIN \ @@ -1410,6 +1412,7 @@ export XWORKSPACE_CONSOLE_PUBLIC_ACCESS="${XWORKSPACE_CONSOLE_PUBLIC_ACCESS:-fal export XWORKMATE_BRIDGE_PUBLIC_ACCESS="${XWORKMATE_BRIDGE_PUBLIC_ACCESS:-true}" export GATEWAY_OPENCLAW_PUBLIC_ACCESS="${GATEWAY_OPENCLAW_PUBLIC_ACCESS:-false}" export VAULT_PUBLIC_ACCESS="${VAULT_PUBLIC_ACCESS:-false}" +export LITELLM_CADDY_CONFIG_ENABLED="${LITELLM_CADDY_CONFIG_ENABLED:-false}" export VAULT_DEPLOY_MODE="${VAULT_DEPLOY_MODE:-standalone}" export XWORKMATE_BRIDGE_VALIDATION_BASE_URL="${XWORKMATE_BRIDGE_VALIDATION_BASE_URL:-http://127.0.0.1:8787}" @@ -1458,6 +1461,7 @@ append_secret_var() { append_var "AI_WORKSPACE_SECURITY_LEVEL" "ai_workspace_security_level" append_var "LITELLM_API_CADDY_STRICT_WHITELIST" "litellm_api_caddy_strict_whitelist" +append_var "LITELLM_CADDY_CONFIG_ENABLED" "litellm_caddy_config_enabled" append_var "XWORKSPACE_CONSOLE_PUBLIC_ACCESS" "xworkspace_console_public_access" append_var "XWORKMATE_BRIDGE_PUBLIC_ACCESS" "xworkmate_bridge_public_access" append_var "XWORKMATE_BRIDGE_DOMAIN" "xworkmate_bridge_domain"