From dcdc9bea7b49f045e1ac0a30f85a5e0c84c1e8db Mon Sep 17 00:00:00 2001 From: Haitao Pan Date: Wed, 3 Jun 2026 10:49:49 +0800 Subject: [PATCH] feat: Remote Desktop Ansible Deployment for xworkmate-bridge --- group_vars/xworkmate_bridge_distributed.yml | 13 + .../vhosts/xworkmate_bridge/defaults/main.yml | 10 + .../README.md | 188 +++++ .../defaults/main.yml | 4 + .../tasks/main.yml | 186 ++++- .../templates/wg-xwm.conf.j2 | 12 +- ...k-wireguard-over-vless-closure-evidence.sh | 129 +++ .../verify-wireguard-over-vless-closure.sh | 748 ++++++++++++++++++ 8 files changed, 1278 insertions(+), 12 deletions(-) create mode 100755 scripts/check-wireguard-over-vless-closure-evidence.sh create mode 100755 scripts/verify-wireguard-over-vless-closure.sh diff --git a/group_vars/xworkmate_bridge_distributed.yml b/group_vars/xworkmate_bridge_distributed.yml index 88bb899..cb77a76 100644 --- a/group_vars/xworkmate_bridge_distributed.yml +++ b/group_vars/xworkmate_bridge_distributed.yml @@ -34,3 +34,16 @@ xworkmate_bridge_distributed_vpn_nodes: wg_ip: 172.29.10.2 public_key: iYlnFaWiMfMelpiN8ZV2SwCDrLihqtJXvHUsM3BN9zU= peer: jp-xhttp-contabo.svc.plus + +xworkmate_bridge_distributed_vpn_clients: + - id: shenlan-macos + wg_ip: 172.29.10.10 + public_key: jfHsw1HIqRQzGvfsRfdkS7BLThDbBvWMsAlJRp1kdkw= + attach_to: + - jp-xhttp-contabo.svc.plus + - cn-xworkmate-bridge.svc.plus + - id: shenlan-ios + wg_ip: 172.29.10.11 + public_key: I/zCL7gLWrY6FZiLXUs7i/vivU5Xuo8r7EbkNhtv12w= + attach_to: + - jp-xhttp-contabo.svc.plus diff --git a/roles/vhosts/xworkmate_bridge/defaults/main.yml b/roles/vhosts/xworkmate_bridge/defaults/main.yml index e13d741..633633e 100644 --- a/roles/vhosts/xworkmate_bridge/defaults/main.yml +++ b/roles/vhosts/xworkmate_bridge/defaults/main.yml @@ -103,3 +103,13 @@ xworkmate_bridge_obsolete_caddy_fragment_paths: xworkmate_bridge_packages: - caddy + - gstreamer1.0-tools + - gstreamer1.0-plugins-base + - gstreamer1.0-plugins-good + - gstreamer1.0-plugins-bad + - gstreamer1.0-plugins-ugly + - gstreamer1.0-libav + - libgstreamer1.0-dev + - libgstreamer-plugins-base1.0-dev + - xdotool + - ffmpeg diff --git a/roles/vhosts/xworkmate_bridge_distributed_vpn/README.md b/roles/vhosts/xworkmate_bridge_distributed_vpn/README.md index 5a80c2f..098cb7b 100644 --- a/roles/vhosts/xworkmate_bridge_distributed_vpn/README.md +++ b/roles/vhosts/xworkmate_bridge_distributed_vpn/README.md @@ -38,6 +38,24 @@ Systemd units: - `xray-wg-tproxy.service` - `xworkmate-bridge-vpn-forwarder.service` +Remote access clients are defined in +`xworkmate_bridge_distributed_vpn_clients`. Each client can set `attach_to` to +control which bridge nodes add the client public key as a direct WireGuard peer. +When a client attaches to only one node, the opposite node adds that client's +`/32` address to the inter-node peer `AllowedIPs` so return traffic routes +through the attached node instead of trying to contact the client directly. + +For the productized overlay path, these client entries are the server-side +projection of the `accounts.svc.plus` overlay contract: + +- `id` maps to `/api/overlay/devices/register` `device_id` +- `public_key` maps to `wireguard_public_key` +- `wg_ip` maps to the `/api/overlay/config` `wireguard.address` without the `/32` + +The client-side CLI renders the matching WireGuard peer with +`Endpoint = 127.0.0.1:51830`; this role keeps the gateway-side peer list in +systemd-managed `/etc/wireguard/wg-xwm.conf`. + The WireGuard peer endpoint on both sides is local: ```ini @@ -70,6 +88,37 @@ xworkmate_bridge_distributed_vpn_vless_port: 2443 xworkmate_bridge_distributed_vpn_forwarder_port: 8787 ``` +Current remote access clients: + +```yaml +xworkmate_bridge_distributed_vpn_clients: + - id: shenlan-macos + wg_ip: 172.29.10.10 + attach_to: + - jp-xhttp-contabo.svc.plus + - cn-xworkmate-bridge.svc.plus + - id: shenlan-ios + wg_ip: 172.29.10.11 + attach_to: + - jp-xhttp-contabo.svc.plus +``` + +`shenlan-ios` is intentionally single-attached to the primary node. XStream-VPN +on iOS owns the System VPN runtime and reaches the private bridge network +through the primary node; the CN edge reaches `172.29.10.11/32` through its +primary inter-node WireGuard peer. + +The current static list is a bootstrap bridge. The next closure step is to let +the Go CLI or control plane export this list from `accounts.svc.plus` instead of +editing `group_vars` by hand. + +The role validates this bootstrap client contract before rendering WireGuard: + +- client `id` values must be present and unique +- client `wg_ip` values must be host IPv4 addresses, unique, and must not reuse a gateway IP +- client `public_key` values must look like WireGuard public keys +- `attach_to` must be non-empty when set and may only reference known VPN node inventory names + ## Secrets This role reads secrets from the Vault service, not from a local Ansible Vault @@ -80,10 +129,27 @@ Required controller environment: ```bash export VAULT_SERVER_URL=https://vault.svc.plus export VAULT_SERVER_ROOT_ACCESS_TOKEN=... +export INTERNAL_SERVICE_TOKEN=... ``` `VAULT_TOKEN` is also accepted when `VAULT_SERVER_ROOT_ACCESS_TOKEN` is not set. Do not commit Vault tokens, WireGuard private keys, or the shared Xray UUID. +The role posts the current gateway node to `accounts.svc.plus` through +`/api/internal/overlay/nodes/heartbeat`; this is required by default because +clients need the gateway `transport_uuid` before `/api/overlay/config` can be +issued. Set `ACCOUNTS_SERVICE_URL` when the accounts service is not +`https://accounts.svc.plus`. For an explicit offline/bootstrap deployment only, +set `xworkmate_bridge_distributed_vpn_sync_accounts_required: false`. +Before posting the heartbeat, the role derives the public key from the +Vault-provided WireGuard private key with `wg pubkey` and checks it against the +inventory public key. It also validates `common.xray_uuid` as a UUID. This keeps +`accounts.svc.plus` from publishing a gateway contract that clients can render +but cannot actually handshake with. +After the heartbeat returns, the role checks the returned `node` payload against +the deployed gateway facts: node id, network id, WireGuard public key/address, +endpoint host/port, `vless-tls` transport, `tls` security, UUID presence, and +healthy state. A successful deploy therefore proves the accounts control plane +accepted the same gateway contract the CLI will later sync. Vault KV base path: @@ -102,6 +168,11 @@ hosts/ The Xray UUID is the shared management-side UUID for this bridge transport. It is not derived from tenant accounts or Xray account sync. +`accounts.svc.plus` must persist the same value as the overlay node +`transport_uuid` through the internal heartbeat API, or expose it as +`OVERLAY_TRANSPORT_UUID` for local/bootstrap use; otherwise +`/api/overlay/config` must not issue client configs because VLESS +authentication would fail at the gateway. ## Bridge Forwarding @@ -134,12 +205,124 @@ Run from the playbooks repo: cd /Users/shenlan/workspaces/cloud-neutral-toolkit/playbooks export VAULT_SERVER_URL=https://vault.svc.plus export VAULT_SERVER_ROOT_ACCESS_TOKEN=... +export INTERNAL_SERVICE_TOKEN=... +export ACCOUNT_EMAIL=... +export ACCOUNT_PASSWORD=... +export BRIDGE_AUTH_TOKEN=... ANSIBLE_CONFIG=ansible.cfg ansible-playbook -i inventory.ini vpn-wireguard-over-vless.yml --check --diff ANSIBLE_CONFIG=ansible.cfg ansible-playbook -i inventory.ini vpn-wireguard-over-vless.yml -f 1 ``` Use `-f 1` for this two-host path when long SSH control sessions are unstable. +For the full accounts + playbooks + local CLI closure, use: + +```bash +scripts/verify-wireguard-over-vless-closure.sh +``` + +Recommended preflight and full run when using the adjacent repo `.env` files: + +```bash +OVERLAY_ENV_FILES=.env,../accounts.svc.plus/.env \ +OVERLAY_BUILD_BIN=1 \ +OVERLAY_CHECK_ONLY=1 \ +scripts/verify-wireguard-over-vless-closure.sh + +OVERLAY_ENV_FILES=.env,../accounts.svc.plus/.env \ +OVERLAY_BUILD_BIN=1 \ +scripts/verify-wireguard-over-vless-closure.sh +``` + +`OVERLAY_CHECK_ONLY=1` is only a prerequisite check. It must not be treated as +closure evidence for account login, device registration, config delivery, local +runtime startup, private connectivity, or config ack. After the full run, verify +the resulting evidence directory: + +```bash +scripts/check-wireguard-over-vless-closure-evidence.sh /tmp/wireguard-over-vless-closure- +``` + +The full closure script also runs this evidence checker before a successful +exit and writes the result to `closure-check.log`. + +The closure script performs the product path in order: + +1. `overlayctl login` +2. `overlayctl register-device` +3. `overlayctl sync-config`, `render`, and `preflight` +4. `overlayctl apply-playbooks-client` +5. `ansible-playbook vpn-wireguard-over-vless.yml` +6. `overlayctl sync-config`, `render`, and `preflight` again after gateway heartbeat +7. `overlayctl up` +8. `overlayctl check-connectivity --bearer "$BRIDGE_AUTH_TOKEN"` +9. `overlayctl ack-config` +10. optional `overlayctl down` when `OVERLAY_TEARDOWN=1` + +Useful overrides: + +- `ACCOUNTS_REPO`: defaults to `../accounts.svc.plus` +- `ACCOUNTS_SERVICE_URL`: defaults to `https://accounts.svc.plus` +- `OVERLAY_NODE_ID`: defaults to `xworkmate-bridge` +- `OVERLAY_ATTACH_TO`: comma-separated gateway inventory hosts +- `OVERLAY_REGISTER_ARGS`: optional explicit args for `register-device`; when + unset, the script reuses `~/.xoverlay/session.json` public/private keys and + only falls back to `--generate-key` when no local keypair exists +- `OVERLAY_STATE_FILE`: defaults to `~/.xoverlay/session.json` +- `OVERLAY_CONFIG_FILE`: defaults to `~/.xoverlay/overlay-config.json` +- `OVERLAY_EVIDENCE_DIR`: defaults to `/tmp/wireguard-over-vless-closure-`; + every run writes `run.log`, `steps.log`, `closure-requirements.tsv`, + `closure-verdict.env`, `summary.env`, `rerun.env`, tool versions, git status, + and redacted overlay state/config snapshots there. Completed full-run evidence + also includes `closure-check.log`. `summary.env` includes the selected + paths, playbooks/accounts Git HEAD and dirty state, build output, + `overlayctl` SHA256, per-requirement closure statuses, `closure_complete`, + last recorded closure step/status, and any missing required paths, tools, or + credentials. The local connectivity check and config ack are reported + separately as `closure_connectivity_status` and `closure_ack_status`. + `closure-requirements.tsv` maps each required closure item to its step and + status; `optional_teardown` is recorded but not required for completion. + `closure-verdict.env` is the machine gate: `closure_ready=1` only when every + required closure item is `ok`, and `required_items_failed` lists each missing + or failed item otherwise. + Use `scripts/check-wireguard-over-vless-closure-evidence.sh ` + to assert the evidence directory is a completed closure; it checks both + `closure-verdict.env` and `summary.env` for a completed state, requires + `summary.env` `status=0`, requires the verdict `requirements_file` to point + at the same evidence directory, and rejects any non-empty + `required_items_failed` value. It also verifies that each required item in + `closure-requirements.tsv` matches the last status for that step in + `steps.log`. + `steps.log` records each completed, skipped, or failed closure phase so a + failed run can be resumed from the right boundary. + `rerun.env` contains only non-secret exports and comments for the next run, + and can be sourced after adding the missing secret values separately. +- `OVERLAY_CAPTURE_LOG=0`: disable teeing stdout/stderr to `run.log` +- `OVERLAYCTL_BIN`: use a prebuilt `overlayctl`; sudo runtime commands will use + the same binary and preserve `HOME` so they read the same local state file +- `OVERLAY_BUILD_BIN=1`: build `overlayctl` from `ACCOUNTS_REPO` into the + evidence directory before checks and use that binary for every CLI step +- `OVERLAY_BUILD_BIN_PATH`: override the build output path; defaults to + `/overlayctl` +- `OVERLAY_ENV_FILES`: comma-separated `.env` files to load before checks. The + script only imports the closure credential keys it understands and records + loaded key names, not values, in `summary.env`. +- `OVERLAY_CHECK_ONLY=1`: check local tools plus required environment and write + evidence without logging in, deploying, or starting the local runtime +- `OVERLAY_USE_SUDO=0`: run `up/status/down` without sudo when the local runtime + is already permissioned +- `OVERLAY_SKIP_LOCAL_TOOL_CHECK=1`: skip the script's early `python3`, `go` or + `OVERLAYCTL_BIN`, `ansible-playbook`, `wg`, `wg-quick`, `xray`, and `sudo` + checks; `overlayctl preflight` still validates runtime tools later +- `OVERLAY_ANSIBLE_SYNTAX_ARGS`: extra args for the syntax-check invocation +- `OVERLAY_ANSIBLE_DEPLOY_ARGS`: extra args for the deploy invocation; defaults + to `-f 1`, and can be set to values such as `--check --diff`, `--limit host`, + or `--tags xworkmate_bridge` +- `OVERLAY_SKIP_DEPLOY=1`: skip playbooks deploy when gateways are already deployed +- `OVERLAY_SKIP_UP=1`: stop after render/preflight without starting local runtime +- `OVERLAY_TEARDOWN=1`: run `overlayctl down` after connectivity and ack +- `OVERLAY_TEARDOWN_ON_ERROR=1`: if a command fails after `overlayctl up` + succeeds, run `overlayctl down` before exiting ## Verification @@ -151,6 +334,11 @@ xray run -test -config /usr/local/etc/xray/wireguard-over-vless.json wg show wg-xwm ``` +The role checks `wg show wg-xwm` after service start. It asserts that the +inter-node peer public key and `/32` route are present, that each client attached +to the current node appears as a peer with its `/32` route, and that clients +attached only to the opposite node are routed through the inter-node peer. + From the primary node: ```bash diff --git a/roles/vhosts/xworkmate_bridge_distributed_vpn/defaults/main.yml b/roles/vhosts/xworkmate_bridge_distributed_vpn/defaults/main.yml index 7e4138a..aa34b76 100644 --- a/roles/vhosts/xworkmate_bridge_distributed_vpn/defaults/main.yml +++ b/roles/vhosts/xworkmate_bridge_distributed_vpn/defaults/main.yml @@ -17,4 +17,8 @@ xworkmate_bridge_distributed_vpn_vault_addr: "{{ lookup('ansible.builtin.env', ' xworkmate_bridge_distributed_vpn_vault_token: "{{ lookup('ansible.builtin.env', 'VAULT_SERVER_ROOT_ACCESS_TOKEN') | default(lookup('ansible.builtin.env', 'VAULT_TOKEN'), true) }}" xworkmate_bridge_distributed_vpn_vault_mount: kv xworkmate_bridge_distributed_vpn_vault_base_path: xworkmate-bridge/distributed/wireguard-over-vless +xworkmate_bridge_distributed_vpn_accounts_url: "{{ lookup('ansible.builtin.env', 'ACCOUNTS_SERVICE_URL') | default('https://accounts.svc.plus', true) }}" +xworkmate_bridge_distributed_vpn_accounts_internal_token: "{{ lookup('ansible.builtin.env', 'INTERNAL_SERVICE_TOKEN') | default('', true) }}" +xworkmate_bridge_distributed_vpn_sync_accounts_required: true xworkmate_bridge_distributed_vpn_nodes: {} +xworkmate_bridge_distributed_vpn_clients: [] diff --git a/roles/vhosts/xworkmate_bridge_distributed_vpn/tasks/main.yml b/roles/vhosts/xworkmate_bridge_distributed_vpn/tasks/main.yml index bd67d68..419eda9 100644 --- a/roles/vhosts/xworkmate_bridge_distributed_vpn/tasks/main.yml +++ b/roles/vhosts/xworkmate_bridge_distributed_vpn/tasks/main.yml @@ -11,6 +11,67 @@ xworkmate_bridge_distributed_vpn_current_node: "{{ xworkmate_bridge_distributed_vpn_nodes[inventory_hostname] }}" xworkmate_bridge_distributed_vpn_peer_node: "{{ xworkmate_bridge_distributed_vpn_nodes[xworkmate_bridge_distributed_vpn_nodes[inventory_hostname].peer] }}" +- name: Assert distributed VPN runtime ports are valid + ansible.builtin.assert: + that: + - xworkmate_bridge_distributed_vpn_vless_port | int >= 1 + - xworkmate_bridge_distributed_vpn_vless_port | int <= 65535 + - xworkmate_bridge_distributed_vpn_wireguard_port | int >= 1 + - xworkmate_bridge_distributed_vpn_wireguard_port | int <= 65535 + - xworkmate_bridge_distributed_vpn_local_tproxy_port | int >= 1 + - xworkmate_bridge_distributed_vpn_local_tproxy_port | int <= 65535 + - xworkmate_bridge_distributed_vpn_forwarder_port | int >= 1 + - xworkmate_bridge_distributed_vpn_forwarder_port | int <= 65535 + fail_msg: "Distributed VPN ports must be between 1 and 65535" + +- name: Assert distributed VPN node contract is valid + ansible.builtin.assert: + that: + - xworkmate_bridge_distributed_vpn_nodes | length > 0 + - xworkmate_bridge_distributed_vpn_nodes.values() | map(attribute='node_id') | list | length == xworkmate_bridge_distributed_vpn_nodes.values() | map(attribute='node_id') | unique | list | length + - xworkmate_bridge_distributed_vpn_nodes.values() | map(attribute='wg_ip') | list | length == xworkmate_bridge_distributed_vpn_nodes.values() | map(attribute='wg_ip') | unique | list | length + - xworkmate_bridge_distributed_vpn_current_node.node_id | trim | length > 0 + - xworkmate_bridge_distributed_vpn_current_node.domain | trim | length > 0 + - xworkmate_bridge_distributed_vpn_current_node.wg_ip is match('^[0-9]{1,3}(\\.[0-9]{1,3}){3}$') + - xworkmate_bridge_distributed_vpn_current_node.public_key is match('^[A-Za-z0-9+/]{43}=$') + - xworkmate_bridge_distributed_vpn_peer_node.wg_ip is match('^[0-9]{1,3}(\\.[0-9]{1,3}){3}$') + - xworkmate_bridge_distributed_vpn_peer_node.public_key is match('^[A-Za-z0-9+/]{43}=$') + fail_msg: "Distributed VPN nodes must have unique IDs/IPs plus valid domain, host IP, and WireGuard public key fields" + +- name: Assert distributed VPN client peer contract is valid + ansible.builtin.assert: + that: + - item.id | trim | length > 0 + - item.wg_ip is match('^[0-9]{1,3}(\\.[0-9]{1,3}){3}$') + - item.wg_ip not in (xworkmate_bridge_distributed_vpn_nodes.values() | map(attribute='wg_ip') | list) + - item.public_key is match('^[A-Za-z0-9+/]{43}=$') + - item.attach_to | default(xworkmate_bridge_distributed_vpn_nodes.keys() | list) | length > 0 + - item.attach_to | default(xworkmate_bridge_distributed_vpn_nodes.keys() | list) | difference(xworkmate_bridge_distributed_vpn_nodes.keys() | list) | length == 0 + fail_msg: "Distributed VPN clients must have id, host wg_ip, valid WireGuard public_key, and attach_to values that reference existing VPN nodes" + loop: "{{ xworkmate_bridge_distributed_vpn_clients }}" + loop_control: + label: "{{ item.id | default('unnamed-client') }}" + +- name: Assert distributed VPN client peer IDs and IPs are unique + ansible.builtin.assert: + that: + - xworkmate_bridge_distributed_vpn_clients | map(attribute='id') | list | length == xworkmate_bridge_distributed_vpn_clients | map(attribute='id') | unique | list | length + - xworkmate_bridge_distributed_vpn_clients | map(attribute='wg_ip') | list | length == xworkmate_bridge_distributed_vpn_clients | map(attribute='wg_ip') | unique | list | length + fail_msg: "Distributed VPN clients must have unique ids and WireGuard IPs" + +- name: Install distributed VPN packages + ansible.builtin.apt: + name: + - ca-certificates + - curl + - unzip + - wireguard-tools + - socat + state: present + update_cache: true + when: + - ansible_os_family == "Debian" + - name: Resolve Caddy-managed TLS certificate paths for VLESS ansible.builtin.set_fact: xworkmate_bridge_distributed_vpn_tls_cert_path: "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/{{ xworkmate_bridge_distributed_vpn_current_node.domain }}/{{ xworkmate_bridge_distributed_vpn_current_node.domain }}.crt" @@ -64,18 +125,79 @@ xworkmate_bridge_distributed_vpn_wireguard_private_key: "{{ xworkmate_bridge_distributed_vpn_host_secret.json.data.data.wireguard_private_key }}" no_log: true -- name: Install distributed VPN packages - ansible.builtin.apt: - name: - - ca-certificates - - curl - - unzip - - wireguard-tools - - socat - state: present - update_cache: true +- name: Derive distributed VPN WireGuard public key from Vault private key + ansible.builtin.command: + cmd: wg pubkey + stdin: "{{ xworkmate_bridge_distributed_vpn_wireguard_private_key }}" + register: xworkmate_bridge_distributed_vpn_derived_public_key + changed_when: false + no_log: true + +- name: Assert distributed VPN secret and inventory keys match + ansible.builtin.assert: + that: + - xworkmate_bridge_distributed_vpn_xray_uuid | trim | regex_search('^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$') + - xworkmate_bridge_distributed_vpn_derived_public_key.stdout | trim == xworkmate_bridge_distributed_vpn_current_node.public_key | trim + fail_msg: "Vault WireGuard private key must match the inventory public key, and common.xray_uuid must be a UUID" + no_log: true + +- name: Assert accounts overlay node sync is configured + ansible.builtin.assert: + that: + - not (xworkmate_bridge_distributed_vpn_sync_accounts_required | bool) or (xworkmate_bridge_distributed_vpn_accounts_internal_token | trim | length > 0) + - not (xworkmate_bridge_distributed_vpn_sync_accounts_required | bool) or (xworkmate_bridge_distributed_vpn_accounts_url | trim | length > 0) + fail_msg: "Set INTERNAL_SERVICE_TOKEN and ACCOUNTS_SERVICE_URL before deploying distributed VPN, or set xworkmate_bridge_distributed_vpn_sync_accounts_required=false for an explicit offline bootstrap" + +- name: Sync distributed VPN overlay node to accounts.svc.plus + ansible.builtin.uri: + url: "{{ xworkmate_bridge_distributed_vpn_accounts_url | trim }}/api/internal/overlay/nodes/heartbeat" + method: POST + headers: + X-Service-Token: "{{ xworkmate_bridge_distributed_vpn_accounts_internal_token }}" + body_format: json + body: + node_id: "{{ xworkmate_bridge_distributed_vpn_current_node.node_id }}" + network_id: xworkmate-private + name: "{{ xworkmate_bridge_distributed_vpn_current_node.node_id }}" + role: "{{ xworkmate_bridge_distributed_vpn_current_node.role | default('gateway') }}" + region: "{{ xworkmate_bridge_distributed_vpn_current_node.region | default('') }}" + wireguard_public_key: "{{ xworkmate_bridge_distributed_vpn_current_node.public_key }}" + wireguard_address: "{{ xworkmate_bridge_distributed_vpn_current_node.wg_ip }}" + endpoint_host: "{{ xworkmate_bridge_distributed_vpn_current_node.domain }}" + endpoint_port: "{{ xworkmate_bridge_distributed_vpn_vless_port | int }}" + transport_type: vless-tls + transport_security: tls + transport_uuid: "{{ xworkmate_bridge_distributed_vpn_xray_uuid }}" + healthy: true + status_code: 200 + timeout: 30 + register: xworkmate_bridge_distributed_vpn_accounts_heartbeat + delegate_to: localhost + become: false + check_mode: false + no_log: true when: - - ansible_os_family == "Debian" + - xworkmate_bridge_distributed_vpn_accounts_internal_token | trim | length > 0 + +- name: Assert accounts overlay node heartbeat matches deployed gateway contract + ansible.builtin.assert: + that: + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.id == xworkmate_bridge_distributed_vpn_current_node.node_id + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.network_id == 'xworkmate-private' + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.wireguard_public_key == xworkmate_bridge_distributed_vpn_current_node.public_key + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.wireguard_address == xworkmate_bridge_distributed_vpn_current_node.wg_ip + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.endpoint_host == xworkmate_bridge_distributed_vpn_current_node.domain + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.endpoint_port | int == xworkmate_bridge_distributed_vpn_vless_port | int + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.transport_type == 'vless-tls' + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.transport_security == 'tls' + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.transport_uuid_set | bool + - xworkmate_bridge_distributed_vpn_accounts_heartbeat.json.node.healthy | bool + fail_msg: "accounts.svc.plus overlay node heartbeat response does not match the deployed gateway contract" + delegate_to: localhost + become: false + no_log: true + when: + - xworkmate_bridge_distributed_vpn_accounts_internal_token | trim | length > 0 - name: Check Xray binary ansible.builtin.stat: @@ -170,6 +292,13 @@ - /etc/wireguard - "{{ xworkmate_bridge_distributed_vpn_xray_config_dir }}" +- name: Enable IPv4 forwarding for distributed WireGuard routing + ansible.builtin.sysctl: + name: net.ipv4.ip_forward + value: "1" + state: present + reload: true + - name: Check VLESS TLS certificate files ansible.builtin.stat: path: "{{ item }}" @@ -284,3 +413,38 @@ changed_when: false when: - not ansible_check_mode + +- name: Assert WireGuard runtime includes the distributed peer route + ansible.builtin.assert: + that: + - xworkmate_bridge_distributed_vpn_peer_node.public_key in xworkmate_bridge_distributed_vpn_wg_show.stdout + - (xworkmate_bridge_distributed_vpn_peer_node.wg_ip ~ '/32') in xworkmate_bridge_distributed_vpn_wg_show.stdout + fail_msg: "WireGuard runtime does not include the distributed peer public key and /32 route" + when: + - not ansible_check_mode + +- name: Assert WireGuard runtime includes attached client peers + ansible.builtin.assert: + that: + - item.public_key in xworkmate_bridge_distributed_vpn_wg_show.stdout + - (item.wg_ip ~ '/32') in xworkmate_bridge_distributed_vpn_wg_show.stdout + fail_msg: "WireGuard runtime does not include an attached client peer public key and /32 route" + loop: "{{ xworkmate_bridge_distributed_vpn_clients }}" + loop_control: + label: "{{ item.id }}" + when: + - not ansible_check_mode + - inventory_hostname in (item.attach_to | default(xworkmate_bridge_distributed_vpn_nodes.keys() | list)) + +- name: Assert WireGuard runtime routes peer-attached clients through the distributed peer + ansible.builtin.assert: + that: + - (item.wg_ip ~ '/32') in xworkmate_bridge_distributed_vpn_wg_show.stdout + fail_msg: "WireGuard runtime does not route a peer-attached client /32 through the distributed peer" + loop: "{{ xworkmate_bridge_distributed_vpn_clients }}" + loop_control: + label: "{{ item.id }}" + when: + - not ansible_check_mode + - inventory_hostname not in (item.attach_to | default(xworkmate_bridge_distributed_vpn_nodes.keys() | list)) + - xworkmate_bridge_distributed_vpn_current_node.peer in (item.attach_to | default(xworkmate_bridge_distributed_vpn_nodes.keys() | list)) diff --git a/roles/vhosts/xworkmate_bridge_distributed_vpn/templates/wg-xwm.conf.j2 b/roles/vhosts/xworkmate_bridge_distributed_vpn/templates/wg-xwm.conf.j2 index 1f0366f..e8bb6d1 100644 --- a/roles/vhosts/xworkmate_bridge_distributed_vpn/templates/wg-xwm.conf.j2 +++ b/roles/vhosts/xworkmate_bridge_distributed_vpn/templates/wg-xwm.conf.j2 @@ -6,6 +6,16 @@ MTU = {{ xworkmate_bridge_distributed_vpn_mtu }} [Peer] PublicKey = {{ xworkmate_bridge_distributed_vpn_peer_node.public_key }} -AllowedIPs = {{ xworkmate_bridge_distributed_vpn_peer_node.wg_ip }}/32 +AllowedIPs = {{ xworkmate_bridge_distributed_vpn_peer_node.wg_ip }}/32{% for client in xworkmate_bridge_distributed_vpn_clients %}{% set attach_to = client.attach_to | default(xworkmate_bridge_distributed_vpn_nodes.keys() | list) %}{% if inventory_hostname not in attach_to and xworkmate_bridge_distributed_vpn_current_node.peer in attach_to %}, {{ client.wg_ip }}/32{% endif %}{% endfor %}{{ '\n' -}} Endpoint = 127.0.0.1:{{ xworkmate_bridge_distributed_vpn_local_tproxy_port }} PersistentKeepalive = {{ xworkmate_bridge_distributed_vpn_persistent_keepalive }} +{% for client in xworkmate_bridge_distributed_vpn_clients %} +{% set attach_to = client.attach_to | default(xworkmate_bridge_distributed_vpn_nodes.keys() | list) %} +{% if inventory_hostname in attach_to %} + +[Peer] +# {{ client.id }} +PublicKey = {{ client.public_key }} +AllowedIPs = {{ client.wg_ip }}/32 +{% endif %} +{% endfor %} diff --git a/scripts/check-wireguard-over-vless-closure-evidence.sh b/scripts/check-wireguard-over-vless-closure-evidence.sh new file mode 100755 index 0000000..a2c5f67 --- /dev/null +++ b/scripts/check-wireguard-over-vless-closure-evidence.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: check-wireguard-over-vless-closure-evidence.sh + +Checks a WireGuard-over-VLESS closure evidence directory produced by +verify-wireguard-over-vless-closure.sh. +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +evidence_dir="${1:-}" +if [[ -z "${evidence_dir}" ]]; then + usage >&2 + exit 2 +fi +if [[ ! -d "${evidence_dir}" ]]; then + echo "evidence directory not found: ${evidence_dir}" >&2 + exit 2 +fi + +verdict_file="${evidence_dir}/closure-verdict.env" +requirements_file="${evidence_dir}/closure-requirements.tsv" +summary_file="${evidence_dir}/summary.env" +steps_file="${evidence_dir}/steps.log" +closure_check_file="${evidence_dir}/closure-check.log" + +missing_files=() +for file in "${verdict_file}" "${requirements_file}" "${summary_file}" "${steps_file}" "${closure_check_file}"; do + if [[ ! -f "${file}" ]]; then + missing_files+=("${file}") + fi +done +if [[ ${#missing_files[@]} -gt 0 ]]; then + printf 'missing evidence file: %s\n' "${missing_files[@]}" >&2 + exit 2 +fi + +closure_ready="$(awk -F '=' '$1 == "closure_ready" { print $2 }' "${verdict_file}" | tail -n 1)" +required_items_failed="$(awk -F '=' '$1 == "required_items_failed" { print $2 }' "${verdict_file}" | tail -n 1)" +verdict_requirements_file="$(awk -F '=' '$1 == "requirements_file" { print $2 }' "${verdict_file}" | tail -n 1)" +summary_status="$(awk -F '=' '$1 == "status" { print $2 }' "${summary_file}" | tail -n 1)" +closure_complete="$(awk -F '=' '$1 == "closure_complete" { print $2 }' "${summary_file}" | tail -n 1)" + +tsv_failed_items="$( + awk -F '\t' ' + NR == 1 { + next + } + $4 == "1" && $3 != "ok" { + print $1 ":" $3 + } + ' "${requirements_file}" | paste -sd ',' - +)" + +step_mismatches="$( + awk -F '\t' ' + FNR == NR { + step_status[$2] = $3 + next + } + FNR == 1 { + next + } + $4 == "1" && step_status[$2] != $3 { + actual = step_status[$2] + if (actual == "") { + actual = "not_run" + } + print $1 ":" $2 ":tsv=" $3 ":steps=" actual + } + ' "${steps_file}" "${requirements_file}" | paste -sd ',' - +)" + +if [[ "${closure_ready}" != "1" ]]; then + echo "closure_ready=${closure_ready:-missing}" >&2 + echo "summary_status=${summary_status:-missing}" >&2 + echo "closure_complete=${closure_complete:-missing}" >&2 + if [[ -n "${required_items_failed}" ]]; then + echo "required_items_failed=${required_items_failed}" >&2 + fi + if [[ -n "${tsv_failed_items}" && "${tsv_failed_items}" != "${required_items_failed}" ]]; then + echo "requirements_tsv_failed=${tsv_failed_items}" >&2 + fi + exit 1 +fi + +if [[ "${summary_status}" != "0" ]]; then + echo "closure verdict says ready, but summary status=${summary_status:-missing}" >&2 + exit 1 +fi + +if [[ "${verdict_requirements_file}" != "${requirements_file}" ]]; then + echo "closure verdict requirements_file does not match evidence directory" >&2 + echo "requirements_file=${verdict_requirements_file:-missing}" >&2 + echo "expected_requirements_file=${requirements_file}" >&2 + exit 1 +fi + +if [[ "${closure_complete}" != "1" ]]; then + echo "closure verdict says ready, but summary closure_complete=${closure_complete:-missing}" >&2 + exit 1 +fi + +if [[ -n "${required_items_failed}" ]]; then + echo "closure verdict says ready, but required_items_failed=${required_items_failed}" >&2 + if [[ -n "${tsv_failed_items}" && "${tsv_failed_items}" != "${required_items_failed}" ]]; then + echo "requirements_tsv_failed=${tsv_failed_items}" >&2 + fi + exit 1 +fi + +if [[ -n "${tsv_failed_items}" ]]; then + echo "closure verdict says ready, but required TSV items are not ok: ${tsv_failed_items}" >&2 + exit 1 +fi + +if [[ -n "${step_mismatches}" ]]; then + echo "closure TSV does not match steps.log: ${step_mismatches}" >&2 + exit 1 +fi + +echo "closure evidence OK: ${evidence_dir}" diff --git a/scripts/verify-wireguard-over-vless-closure.sh b/scripts/verify-wireguard-over-vless-closure.sh new file mode 100755 index 0000000..d2edac6 --- /dev/null +++ b/scripts/verify-wireguard-over-vless-closure.sh @@ -0,0 +1,748 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CLOUD_NEUTRAL_DIR="$(cd "${ROOT_DIR}/.." && pwd)" +ACCOUNTS_REPO="${ACCOUNTS_REPO:-${CLOUD_NEUTRAL_DIR}/accounts.svc.plus}" +ACCOUNTS_SERVICE_URL="${ACCOUNTS_SERVICE_URL:-https://accounts.svc.plus}" +OVERLAY_NODE_ID="${OVERLAY_NODE_ID:-xworkmate-bridge}" +OVERLAY_GROUP_VARS="${OVERLAY_GROUP_VARS:-${ROOT_DIR}/group_vars/xworkmate_bridge_distributed.yml}" +OVERLAY_ATTACH_TO="${OVERLAY_ATTACH_TO:-jp-xhttp-contabo.svc.plus,cn-xworkmate-bridge.svc.plus}" +OVERLAY_REGISTER_ARGS="${OVERLAY_REGISTER_ARGS:-}" +OVERLAY_USE_SUDO="${OVERLAY_USE_SUDO:-1}" +OVERLAY_SKIP_DEPLOY="${OVERLAY_SKIP_DEPLOY:-0}" +OVERLAY_SKIP_UP="${OVERLAY_SKIP_UP:-0}" +OVERLAY_TEARDOWN="${OVERLAY_TEARDOWN:-0}" +OVERLAY_TEARDOWN_ON_ERROR="${OVERLAY_TEARDOWN_ON_ERROR:-0}" +OVERLAY_STATE_FILE="${OVERLAY_STATE_FILE:-${HOME}/.xoverlay/session.json}" +OVERLAY_SKIP_LOCAL_TOOL_CHECK="${OVERLAY_SKIP_LOCAL_TOOL_CHECK:-0}" +OVERLAY_ANSIBLE_SYNTAX_ARGS="${OVERLAY_ANSIBLE_SYNTAX_ARGS:-}" +OVERLAY_ANSIBLE_DEPLOY_ARGS="${OVERLAY_ANSIBLE_DEPLOY_ARGS:--f 1}" +OVERLAY_CONFIG_FILE="${OVERLAY_CONFIG_FILE:-${HOME}/.xoverlay/overlay-config.json}" +OVERLAY_EVIDENCE_DIR="${OVERLAY_EVIDENCE_DIR:-/tmp/wireguard-over-vless-closure-$(date -u +%Y%m%dT%H%M%SZ)}" +OVERLAY_CAPTURE_LOG="${OVERLAY_CAPTURE_LOG:-1}" +OVERLAY_CHECK_ONLY="${OVERLAY_CHECK_ONLY:-0}" +OVERLAY_BUILD_BIN="${OVERLAY_BUILD_BIN:-0}" +OVERLAY_BUILD_BIN_PATH="${OVERLAY_BUILD_BIN_PATH:-${OVERLAY_EVIDENCE_DIR}/overlayctl}" +OVERLAY_ENV_FILES="${OVERLAY_ENV_FILES:-}" +overlay_runtime_started=0 +missing_required_envs=() +missing_required_tools=() +missing_required_paths=() +loaded_env_keys=() +loaded_env_files=() +active_step="" +if [[ "${OVERLAY_CAPTURE_LOG}" == "1" ]]; then + mkdir -p "${OVERLAY_EVIDENCE_DIR}" + run_log_fifo="${OVERLAY_EVIDENCE_DIR}/.run-log.fifo" + rm -f "${run_log_fifo}" + mkfifo "${run_log_fifo}" + tee -a "${OVERLAY_EVIDENCE_DIR}/run.log" < "${run_log_fifo}" & + run_log_tee_pid=$! + exec 3>&1 4>&2 + exec > "${run_log_fifo}" 2>&1 + rm -f "${run_log_fifo}" +fi + +sha256_file() { + local path="$1" + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "${path}" | awk '{print $1}' + return + fi + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "${path}" | awk '{print $1}' + return + fi + return 1 +} + +git_head() { + local repo="$1" + git -C "${repo}" rev-parse HEAD 2>/dev/null || true +} + +git_dirty() { + local repo="$1" + if ! git -C "${repo}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "unknown" + return + fi + if [[ -n "$(git -C "${repo}" status --short 2>/dev/null)" ]]; then + echo "1" + return + fi + echo "0" +} + +is_overlay_env_key() { + case "$1" in + ACCOUNT_EMAIL|ACCOUNT_PASSWORD|BRIDGE_AUTH_TOKEN|VAULT_SERVER_ROOT_ACCESS_TOKEN|VAULT_TOKEN|INTERNAL_SERVICE_TOKEN) + return 0 + ;; + *) + return 1 + ;; + esac +} + +trim_quotes() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + if [[ "${value}" == \"*\" && "${value}" == *\" ]]; then + value="${value:1:${#value}-2}" + elif [[ "${value}" == \'*\' && "${value}" == *\' ]]; then + value="${value:1:${#value}-2}" + fi + printf '%s' "${value}" +} + +shell_quote() { + printf "%q" "$1" +} + +mark_step() { + local step="$1" + local status="$2" + mkdir -p "${OVERLAY_EVIDENCE_DIR}" + printf '%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${step}" "${status}" >> "${OVERLAY_EVIDENCE_DIR}/steps.log" +} + +begin_step() { + active_step="$1" +} + +complete_step() { + local step="$1" + local status="$2" + mark_step "${step}" "${status}" + if [[ "${active_step}" == "${step}" ]]; then + active_step="" + fi +} + +step_status() { + local step="$1" + if [[ ! -f "${OVERLAY_EVIDENCE_DIR}/steps.log" ]]; then + echo "not_run" + return + fi + awk -F '\t' -v step="${step}" '$2 == step { status = $3 } END { if (status == "") { print "not_run" } else { print status } }' "${OVERLAY_EVIDENCE_DIR}/steps.log" +} + +write_closure_requirements() { + local output="$1" + local account_login_status="$2" + local device_register_status="$3" + local config_initial_status="$4" + local playbooks_projection_status="$5" + local gateway_deploy_status="$6" + local config_refresh_status="$7" + local local_runtime_status="$8" + local connectivity_status="$9" + local ack_status="${10}" + { + printf 'requirement\tstep\tstatus\trequired_for_completion\n' + printf 'account_login\t01.login\t%s\t1\n' "${account_login_status}" + printf 'device_registration\t02.register_device\t%s\t1\n' "${device_register_status}" + printf 'initial_config_sync_render_preflight\t03.initial_sync_render_preflight\t%s\t1\n' "${config_initial_status}" + printf 'playbooks_client_projection\t04.apply_playbooks_client\t%s\t1\n' "${playbooks_projection_status}" + printf 'gateway_deploy_and_heartbeat\t05.deploy_gateway\t%s\t1\n' "${gateway_deploy_status}" + printf 'post_heartbeat_config_refresh\t06.refresh_sync_render_preflight\t%s\t1\n' "${config_refresh_status}" + printf 'local_runtime_up\t07.local_runtime_up\t%s\t1\n' "${local_runtime_status}" + printf 'private_connectivity\t08.connectivity\t%s\t1\n' "${connectivity_status}" + printf 'config_ack\t09.ack_config\t%s\t1\n' "${ack_status}" + printf 'optional_teardown\t10.teardown\t%s\t0\n' "$(step_status "10.teardown")" + } > "${output}" +} + +closure_complete_value() { + local account_login_status="$1" + local device_register_status="$2" + local config_initial_status="$3" + local playbooks_projection_status="$4" + local gateway_deploy_status="$5" + local config_refresh_status="$6" + local local_runtime_status="$7" + local connectivity_status="$8" + local ack_status="$9" + if [[ "${account_login_status}" == "ok" \ + && "${device_register_status}" == "ok" \ + && "${config_initial_status}" == "ok" \ + && "${playbooks_projection_status}" == "ok" \ + && "${gateway_deploy_status}" == "ok" \ + && "${config_refresh_status}" == "ok" \ + && "${local_runtime_status}" == "ok" \ + && "${connectivity_status}" == "ok" \ + && "${ack_status}" == "ok" ]]; then + echo "1" + return + fi + echo "0" +} + +write_closure_verdict() { + local output="$1" + local complete="$2" + local failed_items + failed_items="$(awk -F '\t' 'NR > 1 && $4 == "1" && $3 != "ok" { items = items ? items "," $1 ":" $3 : $1 ":" $3 } END { print items }' "${OVERLAY_EVIDENCE_DIR}/closure-requirements.tsv")" + { + echo "closure_ready=${complete}" + echo "required_items_failed=${failed_items}" + echo "requirements_file=${OVERLAY_EVIDENCE_DIR}/closure-requirements.tsv" + } > "${output}" +} + +load_env_files() { + if [[ -z "${OVERLAY_ENV_FILES}" ]]; then + return + fi + local env_file line key value + IFS=',' read -r -a env_files <<< "${OVERLAY_ENV_FILES}" + for env_file in "${env_files[@]}"; do + env_file="${env_file#"${env_file%%[![:space:]]*}"}" + env_file="${env_file%"${env_file##*[![:space:]]}"}" + if [[ -z "${env_file}" ]]; then + continue + fi + if [[ ! -f "${env_file}" ]]; then + missing_required_paths+=("overlay_env_file:${env_file}") + continue + fi + loaded_env_files+=("${env_file}") + while IFS= read -r line || [[ -n "${line}" ]]; do + line="${line#"${line%%[![:space:]]*}"}" + [[ -z "${line}" || "${line}" == \#* ]] && continue + [[ "${line}" == export\ * ]] && line="${line#export }" + [[ "${line}" != *=* ]] && continue + key="${line%%=*}" + value="${line#*=}" + key="${key#"${key%%[![:space:]]*}"}" + key="${key%"${key##*[![:space:]]}"}" + if [[ "${key}" == "${line}" || -z "${key}" ]]; then + continue + fi + if ! is_overlay_env_key "${key}"; then + continue + fi + if [[ -n "${!key:-}" ]]; then + continue + fi + value="$(trim_quotes "${value}")" + if [[ -z "${value}" ]]; then + continue + fi + export "${key}=${value}" + loaded_env_keys+=("${key}@${env_file}") + done < "${env_file}" + done +} + +write_evidence() { + local status="$1" + mkdir -p "${OVERLAY_EVIDENCE_DIR}" + { + echo "status=${status}" + echo "timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "accounts_repo=${ACCOUNTS_REPO}" + echo "accounts_git_head=$(git_head "${ACCOUNTS_REPO}")" + echo "accounts_git_dirty=$(git_dirty "${ACCOUNTS_REPO}")" + echo "playbooks_git_head=$(git_head "${ROOT_DIR}")" + echo "playbooks_git_dirty=$(git_dirty "${ROOT_DIR}")" + echo "accounts_service_url=${ACCOUNTS_SERVICE_URL}" + echo "overlay_node_id=${OVERLAY_NODE_ID}" + echo "overlay_group_vars=${OVERLAY_GROUP_VARS}" + echo "overlay_attach_to=${OVERLAY_ATTACH_TO}" + echo "overlay_skip_deploy=${OVERLAY_SKIP_DEPLOY}" + echo "overlay_skip_up=${OVERLAY_SKIP_UP}" + echo "overlay_use_sudo=${OVERLAY_USE_SUDO}" + echo "overlay_teardown=${OVERLAY_TEARDOWN}" + echo "overlay_teardown_on_error=${OVERLAY_TEARDOWN_ON_ERROR}" + echo "overlay_state_file=${OVERLAY_STATE_FILE}" + echo "overlay_config_file=${OVERLAY_CONFIG_FILE}" + echo "overlay_check_only=${OVERLAY_CHECK_ONLY}" + echo "overlay_build_bin=${OVERLAY_BUILD_BIN}" + echo "overlay_build_bin_path=${OVERLAY_BUILD_BIN_PATH}" + echo "overlay_env_files=${OVERLAY_ENV_FILES}" + echo "overlayctl_bin=${OVERLAYCTL_BIN:-}" + local account_login_status device_register_status config_initial_status playbooks_projection_status + local gateway_deploy_status config_refresh_status local_runtime_status connectivity_status ack_status + account_login_status="$(step_status "01.login")" + device_register_status="$(step_status "02.register_device")" + config_initial_status="$(step_status "03.initial_sync_render_preflight")" + playbooks_projection_status="$(step_status "04.apply_playbooks_client")" + gateway_deploy_status="$(step_status "05.deploy_gateway")" + config_refresh_status="$(step_status "06.refresh_sync_render_preflight")" + local_runtime_status="$(step_status "07.local_runtime_up")" + connectivity_status="$(step_status "08.connectivity")" + ack_status="$(step_status "09.ack_config")" + local closure_complete + closure_complete="$(closure_complete_value \ + "${account_login_status}" \ + "${device_register_status}" \ + "${config_initial_status}" \ + "${playbooks_projection_status}" \ + "${gateway_deploy_status}" \ + "${config_refresh_status}" \ + "${local_runtime_status}" \ + "${connectivity_status}" \ + "${ack_status}")" + write_closure_requirements \ + "${OVERLAY_EVIDENCE_DIR}/closure-requirements.tsv" \ + "${account_login_status}" \ + "${device_register_status}" \ + "${config_initial_status}" \ + "${playbooks_projection_status}" \ + "${gateway_deploy_status}" \ + "${config_refresh_status}" \ + "${local_runtime_status}" \ + "${connectivity_status}" \ + "${ack_status}" + write_closure_verdict "${OVERLAY_EVIDENCE_DIR}/closure-verdict.env" "${closure_complete}" + echo "closure_account_login_status=${account_login_status}" + echo "closure_device_register_status=${device_register_status}" + echo "closure_config_initial_status=${config_initial_status}" + echo "closure_playbooks_projection_status=${playbooks_projection_status}" + echo "closure_gateway_deploy_status=${gateway_deploy_status}" + echo "closure_config_refresh_status=${config_refresh_status}" + echo "closure_local_runtime_status=${local_runtime_status}" + echo "closure_connectivity_status=${connectivity_status}" + echo "closure_ack_status=${ack_status}" + echo "closure_complete=${closure_complete}" + if [[ -s "${OVERLAY_EVIDENCE_DIR}/steps.log" ]]; then + local last_step_timestamp last_step last_step_status + IFS=$'\t' read -r last_step_timestamp last_step last_step_status < <(tail -n 1 "${OVERLAY_EVIDENCE_DIR}/steps.log") + echo "last_step_timestamp=${last_step_timestamp}" + echo "last_step=${last_step}" + echo "last_step_status=${last_step_status}" + fi + if [[ ${#loaded_env_files[@]} -gt 0 ]]; then + local joined_loaded_env_files + joined_loaded_env_files="$(IFS=','; echo "${loaded_env_files[*]}")" + echo "loaded_env_files=${joined_loaded_env_files}" + fi + if [[ ${#loaded_env_keys[@]} -gt 0 ]]; then + local joined_loaded_env_keys + joined_loaded_env_keys="$(IFS=','; echo "${loaded_env_keys[*]}")" + echo "loaded_env_keys=${joined_loaded_env_keys}" + fi + if [[ -n "${OVERLAYCTL_BIN:-}" && -x "${OVERLAYCTL_BIN}" ]]; then + overlayctl_sha256="$(sha256_file "${OVERLAYCTL_BIN}" 2>/dev/null || true)" + if [[ -n "${overlayctl_sha256}" ]]; then + echo "overlayctl_sha256=${overlayctl_sha256}" + fi + fi + if [[ ${#missing_required_envs[@]} -gt 0 ]]; then + local joined_missing_envs + joined_missing_envs="$(IFS=','; echo "${missing_required_envs[*]}")" + echo "missing_required_envs=${joined_missing_envs}" + fi + if [[ ${#missing_required_tools[@]} -gt 0 ]]; then + local joined_missing_tools + joined_missing_tools="$(IFS=','; echo "${missing_required_tools[*]}")" + echo "missing_required_tools=${joined_missing_tools}" + fi + if [[ ${#missing_required_paths[@]} -gt 0 ]]; then + local joined_missing_paths + joined_missing_paths="$(IFS=','; echo "${missing_required_paths[*]}")" + echo "missing_required_paths=${joined_missing_paths}" + fi + } > "${OVERLAY_EVIDENCE_DIR}/summary.env" + { + echo "# Non-secret settings for rerunning the closure script." + echo "# Source or copy these exports, then provide the missing secret values separately." + echo "export ACCOUNTS_REPO=$(shell_quote "${ACCOUNTS_REPO}")" + echo "export ACCOUNTS_SERVICE_URL=$(shell_quote "${ACCOUNTS_SERVICE_URL}")" + echo "export OVERLAY_NODE_ID=$(shell_quote "${OVERLAY_NODE_ID}")" + echo "export OVERLAY_GROUP_VARS=$(shell_quote "${OVERLAY_GROUP_VARS}")" + echo "export OVERLAY_ATTACH_TO=$(shell_quote "${OVERLAY_ATTACH_TO}")" + echo "export OVERLAY_USE_SUDO=$(shell_quote "${OVERLAY_USE_SUDO}")" + echo "export OVERLAY_SKIP_DEPLOY=$(shell_quote "${OVERLAY_SKIP_DEPLOY}")" + echo "export OVERLAY_SKIP_UP=$(shell_quote "${OVERLAY_SKIP_UP}")" + echo "export OVERLAY_TEARDOWN=$(shell_quote "${OVERLAY_TEARDOWN}")" + echo "export OVERLAY_TEARDOWN_ON_ERROR=$(shell_quote "${OVERLAY_TEARDOWN_ON_ERROR}")" + echo "export OVERLAY_STATE_FILE=$(shell_quote "${OVERLAY_STATE_FILE}")" + echo "export OVERLAY_CONFIG_FILE=$(shell_quote "${OVERLAY_CONFIG_FILE}")" + echo "export OVERLAY_ENV_FILES=$(shell_quote "${OVERLAY_ENV_FILES}")" + echo "export OVERLAY_BUILD_BIN=$(shell_quote "${OVERLAY_BUILD_BIN}")" + echo "# OVERLAY_BUILD_BIN_PATH was $(shell_quote "${OVERLAY_BUILD_BIN_PATH}")" + echo "# Leave it unset to build into the next run's evidence directory." + if [[ ${#missing_required_envs[@]} -gt 0 ]]; then + local joined_missing_envs + joined_missing_envs="$(IFS=','; echo "${missing_required_envs[*]}")" + echo "# missing_required_envs=${joined_missing_envs}" + fi + echo "# Recommended preflight:" + echo "# OVERLAY_CHECK_ONLY=1 scripts/verify-wireguard-over-vless-closure.sh" + echo "# Recommended full run:" + echo "# scripts/verify-wireguard-over-vless-closure.sh" + } > "${OVERLAY_EVIDENCE_DIR}/rerun.env" + { + command -v go >/dev/null 2>&1 && go version 2>/dev/null || true + command -v ansible-playbook >/dev/null 2>&1 && ansible-playbook --version 2>/dev/null | head -n 1 || true + command -v wg >/dev/null 2>&1 && wg --version 2>/dev/null || true + command -v wg-quick >/dev/null 2>&1 && echo "wg-quick $(command -v wg-quick)" || true + command -v xray >/dev/null 2>&1 && xray version 2>/dev/null | head -n 1 || true + if [[ -n "${OVERLAYCTL_BIN:-}" ]]; then + echo "overlayctl ${OVERLAYCTL_BIN}" + overlayctl_sha256="$(sha256_file "${OVERLAYCTL_BIN}" 2>/dev/null || true)" + if [[ -n "${overlayctl_sha256}" ]]; then + echo "overlayctl-sha256 ${overlayctl_sha256}" + fi + "${OVERLAYCTL_BIN}" --help 2>/dev/null | head -n 1 || true + else + echo "overlayctl go-run ${ACCOUNTS_REPO}/cmd/overlayctl" + fi + } > "${OVERLAY_EVIDENCE_DIR}/tool-versions.txt" + git -C "${ROOT_DIR}" status --short --branch > "${OVERLAY_EVIDENCE_DIR}/playbooks-git-status.txt" 2>/dev/null || true + git -C "${ACCOUNTS_REPO}" status --short --branch > "${OVERLAY_EVIDENCE_DIR}/accounts-git-status.txt" 2>/dev/null || true + if [[ -f "${OVERLAY_STATE_FILE}" ]]; then + python3 - "${OVERLAY_STATE_FILE}" "${OVERLAY_EVIDENCE_DIR}/state-redacted.json" <<'PY' +import json, sys +data = json.load(open(sys.argv[1])) +for key in ("token", "wireguard_private_key"): + if key in data: + data[key] = "" +print(json.dumps(data, indent=2, sort_keys=True), file=open(sys.argv[2], "w")) +PY + fi + if [[ -f "${OVERLAY_CONFIG_FILE}" ]]; then + python3 - "${OVERLAY_CONFIG_FILE}" "${OVERLAY_EVIDENCE_DIR}/config-redacted.json" <<'PY' +import json, sys +data = json.load(open(sys.argv[1])) +transport = data.get("transport") +if isinstance(transport, dict) and "uuid" in transport: + transport["uuid"] = "" +print(json.dumps(data, indent=2, sort_keys=True), file=open(sys.argv[2], "w")) +PY + fi +} + +check_local_tools() { + missing_required_tools=() + if ! command -v python3 >/dev/null 2>&1; then + missing_required_tools+=("python3") + fi + if [[ -n "${OVERLAYCTL_BIN:-}" ]]; then + if [[ ! -x "${OVERLAYCTL_BIN}" ]]; then + missing_required_tools+=("OVERLAYCTL_BIN executable:${OVERLAYCTL_BIN}") + fi + else + if ! command -v go >/dev/null 2>&1; then + missing_required_tools+=("go") + fi + fi + if [[ "${OVERLAY_SKIP_DEPLOY}" != "1" ]]; then + if ! command -v ansible-playbook >/dev/null 2>&1; then + missing_required_tools+=("ansible-playbook") + fi + fi + if [[ "${OVERLAY_SKIP_UP}" != "1" ]]; then + if ! command -v wg >/dev/null 2>&1; then + missing_required_tools+=("wg") + fi + if ! command -v wg-quick >/dev/null 2>&1; then + missing_required_tools+=("wg-quick") + fi + if ! command -v xray >/dev/null 2>&1; then + missing_required_tools+=("xray") + fi + if [[ "${OVERLAY_USE_SUDO}" == "1" ]]; then + if ! command -v sudo >/dev/null 2>&1; then + missing_required_tools+=("sudo") + fi + fi + fi + if [[ ${#missing_required_tools[@]} -gt 0 ]]; then + local name + for name in "${missing_required_tools[@]}"; do + echo "missing required tool: ${name}" >&2 + done + mark_step "preflight.tools" "failed" + exit 2 + fi + mark_step "preflight.tools" "ok" +} + +check_required_paths() { + missing_required_paths=() + if [[ ! -d "${ACCOUNTS_REPO}" ]]; then + missing_required_paths+=("accounts_repo:${ACCOUNTS_REPO}") + fi + if [[ ! -f "${OVERLAY_GROUP_VARS}" ]]; then + missing_required_paths+=("overlay_group_vars:${OVERLAY_GROUP_VARS}") + fi +} + +report_missing_required_paths() { + if [[ ${#missing_required_paths[@]} -gt 0 ]]; then + local path + for path in "${missing_required_paths[@]}"; do + echo "missing required path: ${path}" >&2 + done + mark_step "preflight.paths" "failed" + exit 2 + fi + mark_step "preflight.paths" "ok" +} + +build_overlayctl_bin() { + if [[ "${OVERLAY_BUILD_BIN}" != "1" ]]; then + return + fi + if [[ -n "${OVERLAYCTL_BIN:-}" ]]; then + echo "using explicit OVERLAYCTL_BIN=${OVERLAYCTL_BIN}; skipping overlayctl build" + return + fi + if ! command -v go >/dev/null 2>&1; then + missing_required_tools+=("go") + echo "missing required tool: go" >&2 + mark_step "overlayctl.build" "failed" + exit 2 + fi + mkdir -p "$(dirname "${OVERLAY_BUILD_BIN_PATH}")" + if [[ ! -d "${ACCOUNTS_REPO}/cmd/overlayctl" ]]; then + missing_required_paths+=("overlayctl_package:${ACCOUNTS_REPO}/cmd/overlayctl") + echo "missing required path: overlayctl_package:${ACCOUNTS_REPO}/cmd/overlayctl" >&2 + mark_step "overlayctl.build" "failed" + exit 2 + fi + echo "building overlayctl from ${ACCOUNTS_REPO} to ${OVERLAY_BUILD_BIN_PATH}" + ( + cd "${ACCOUNTS_REPO}" + CGO_ENABLED=0 go build -trimpath -o "${OVERLAY_BUILD_BIN_PATH}" ./cmd/overlayctl + ) + OVERLAYCTL_BIN="${OVERLAY_BUILD_BIN_PATH}" + export OVERLAYCTL_BIN + mark_step "overlayctl.build" "ok" +} + +check_required_environment() { + missing_required_envs=() + local name + for name in ACCOUNT_EMAIL ACCOUNT_PASSWORD BRIDGE_AUTH_TOKEN; do + if [[ -z "${!name:-}" ]]; then + missing_required_envs+=("${name}") + fi + done + if [[ "${OVERLAY_SKIP_DEPLOY}" != "1" ]]; then + if [[ -z "${VAULT_SERVER_ROOT_ACCESS_TOKEN:-}" && -z "${VAULT_TOKEN:-}" ]]; then + missing_required_envs+=("VAULT_SERVER_ROOT_ACCESS_TOKEN or VAULT_TOKEN") + fi + if [[ -z "${INTERNAL_SERVICE_TOKEN:-}" ]]; then + missing_required_envs+=("INTERNAL_SERVICE_TOKEN") + fi + fi + if [[ ${#missing_required_envs[@]} -gt 0 ]]; then + for name in "${missing_required_envs[@]}"; do + echo "missing required environment: ${name}" >&2 + done + mark_step "preflight.environment" "failed" + exit 2 + fi + mark_step "preflight.environment" "ok" +} + +run_overlayctl() { + if [[ -n "${OVERLAYCTL_BIN:-}" ]]; then + "${OVERLAYCTL_BIN}" "$@" + return + fi + (cd "${ACCOUNTS_REPO}" && go run ./cmd/overlayctl "$@") +} + +run_overlayctl_root() { + if [[ "${OVERLAY_USE_SUDO}" == "1" ]]; then + if [[ -n "${OVERLAYCTL_BIN:-}" ]]; then + sudo -E env HOME="${HOME}" "${OVERLAYCTL_BIN}" "$@" + return + fi + sudo -E env HOME="${HOME}" bash -c 'cd "$1" && shift && "$@"' bash "${ACCOUNTS_REPO}" go run ./cmd/overlayctl "$@" + return + fi + run_overlayctl "$@" +} + +cleanup_on_exit() { + local status=$? + if [[ ${status} -ne 0 && -n "${active_step}" ]]; then + mark_step "${active_step}" "failed" + active_step="" + fi + write_evidence "${status}" + if [[ ${status} -eq 0 && "${OVERLAY_CHECK_ONLY}" != "1" ]]; then + local evidence_checker="${ROOT_DIR}/scripts/check-wireguard-over-vless-closure-evidence.sh" + if [[ -x "${evidence_checker}" ]]; then + echo "closure evidence check pending" > "${OVERLAY_EVIDENCE_DIR}/closure-check.log" + if ! "${evidence_checker}" "${OVERLAY_EVIDENCE_DIR}" > "${OVERLAY_EVIDENCE_DIR}/closure-check.log" 2>&1; then + cat "${OVERLAY_EVIDENCE_DIR}/closure-check.log" >&2 + status=1 + fi + else + echo "missing evidence checker: ${evidence_checker}" | tee "${OVERLAY_EVIDENCE_DIR}/closure-check.log" >&2 + status=1 + fi + fi + local evidence_message="closure evidence: ${OVERLAY_EVIDENCE_DIR}" + echo "${evidence_message}" >&2 + if [[ ${status} -ne 0 && "${OVERLAY_TEARDOWN_ON_ERROR}" == "1" && "${overlay_runtime_started}" == "1" ]]; then + echo "closure failed after local runtime start; running overlayctl down because OVERLAY_TEARDOWN_ON_ERROR=1" >&2 + set +e + run_overlayctl_root down + fi + if [[ -n "${run_log_tee_pid:-}" ]]; then + exec 1>&3 2>&4 + exec 3>&- 4>&- + wait "${run_log_tee_pid}" 2>/dev/null || true + fi + exit "${status}" +} + +trap cleanup_on_exit EXIT + +state_value() { + local key="$1" + if [[ ! -f "${OVERLAY_STATE_FILE}" ]]; then + return 0 + fi + python3 -c 'import json, sys; print(str(json.load(open(sys.argv[1])).get(sys.argv[2], "")).strip())' "${OVERLAY_STATE_FILE}" "${key}" +} + +attach_args=() +IFS=',' read -r -a attach_hosts <<< "${OVERLAY_ATTACH_TO}" +for host in "${attach_hosts[@]}"; do + host="${host#"${host%%[![:space:]]*}"}" + host="${host%"${host##*[![:space:]]}"}" + if [[ -n "${host}" ]]; then + attach_args+=(--attach-to "${host}") + fi +done + +register_args=() +if [[ -n "${OVERLAY_REGISTER_ARGS}" ]]; then + read -r -a register_args <<< "${OVERLAY_REGISTER_ARGS}" +fi +ansible_syntax_args=() +if [[ -n "${OVERLAY_ANSIBLE_SYNTAX_ARGS}" ]]; then + read -r -a ansible_syntax_args <<< "${OVERLAY_ANSIBLE_SYNTAX_ARGS}" +fi +ansible_deploy_args=() +if [[ -n "${OVERLAY_ANSIBLE_DEPLOY_ARGS}" ]]; then + read -r -a ansible_deploy_args <<< "${OVERLAY_ANSIBLE_DEPLOY_ARGS}" +fi + +check_required_paths +load_env_files +report_missing_required_paths +build_overlayctl_bin +if [[ "${OVERLAY_SKIP_LOCAL_TOOL_CHECK}" != "1" ]]; then + check_local_tools +else + mark_step "preflight.tools" "skipped" +fi + +check_required_environment +if [[ "${OVERLAY_CHECK_ONLY}" == "1" ]]; then + mark_step "check_only" "ok" + echo "closure prerequisites satisfied; exiting because OVERLAY_CHECK_ONLY=1" + exit 0 +fi + +echo "[1/10] login to ${ACCOUNTS_SERVICE_URL}" +begin_step "01.login" +run_overlayctl login \ + --server "${ACCOUNTS_SERVICE_URL}" \ + --email "${ACCOUNT_EMAIL}" \ + --password "${ACCOUNT_PASSWORD}" +complete_step "01.login" "ok" + +echo "[2/10] register local WireGuard device" +begin_step "02.register_device" +if [[ ${#register_args[@]} -eq 0 ]]; then + existing_public_key="$(state_value wireguard_public_key)" + existing_private_key="$(state_value wireguard_private_key)" + if [[ -n "${existing_public_key}" && -n "${existing_private_key}" ]]; then + echo "Reusing WireGuard keypair from ${OVERLAY_STATE_FILE}" + register_args+=(--public-key "${existing_public_key}" --private-key "${existing_private_key}") + else + echo "No existing WireGuard keypair found in ${OVERLAY_STATE_FILE}; generating a new one" + register_args+=(--generate-key) + fi +fi +run_overlayctl register-device "${register_args[@]}" +complete_step "02.register_device" "ok" + +echo "[3/10] sync and render initial overlay config" +begin_step "03.initial_sync_render_preflight" +run_overlayctl sync-config --node-id "${OVERLAY_NODE_ID}" +run_overlayctl render +run_overlayctl preflight +complete_step "03.initial_sync_render_preflight" "ok" + +echo "[4/10] project local device into playbooks client peers" +begin_step "04.apply_playbooks_client" +run_overlayctl apply-playbooks-client \ + --group-vars "${OVERLAY_GROUP_VARS}" \ + "${attach_args[@]}" +complete_step "04.apply_playbooks_client" "ok" + +if [[ "${OVERLAY_SKIP_DEPLOY}" != "1" ]]; then + echo "[5/10] deploy WireGuard-over-VLESS gateway path" + begin_step "05.deploy_gateway" + ( + cd "${ROOT_DIR}" + ANSIBLE_CONFIG=ansible.cfg ansible-playbook -i inventory.ini vpn-wireguard-over-vless.yml --syntax-check "${ansible_syntax_args[@]}" + ANSIBLE_CONFIG=ansible.cfg ansible-playbook -i inventory.ini vpn-wireguard-over-vless.yml "${ansible_deploy_args[@]}" + ) + complete_step "05.deploy_gateway" "ok" +else + echo "[5/10] skipped deploy because OVERLAY_SKIP_DEPLOY=1" + mark_step "05.deploy_gateway" "skipped" +fi + +echo "[6/10] refresh config after gateway heartbeat" +begin_step "06.refresh_sync_render_preflight" +run_overlayctl sync-config --node-id "${OVERLAY_NODE_ID}" +run_overlayctl render +run_overlayctl preflight +complete_step "06.refresh_sync_render_preflight" "ok" + +if [[ "${OVERLAY_SKIP_UP}" != "1" ]]; then + echo "[7/10] start local Xray and WireGuard" + begin_step "07.local_runtime_up" + run_overlayctl_root up + overlay_runtime_started=1 + run_overlayctl_root status + complete_step "07.local_runtime_up" "ok" + + echo "[8/10] verify private bridge connectivity" + begin_step "08.connectivity" + run_overlayctl check-connectivity --bearer "${BRIDGE_AUTH_TOKEN}" + complete_step "08.connectivity" "ok" + + echo "[9/10] acknowledge applied overlay config" + begin_step "09.ack_config" + run_overlayctl ack-config + complete_step "09.ack_config" "ok" + + if [[ "${OVERLAY_TEARDOWN}" == "1" ]]; then + echo "[10/10] tear down local overlay runtime" + begin_step "10.teardown" + run_overlayctl_root down + overlay_runtime_started=0 + complete_step "10.teardown" "ok" + else + echo "[10/10] leaving local overlay runtime up; set OVERLAY_TEARDOWN=1 to tear it down" + mark_step "10.teardown" "skipped_runtime_left_up" + fi +else + echo "[7/10] skipped local runtime start because OVERLAY_SKIP_UP=1" + echo "[8/10] skipped connectivity check because OVERLAY_SKIP_UP=1" + echo "[9/10] skipped config ack because OVERLAY_SKIP_UP=1" + echo "[10/10] skipped teardown because OVERLAY_SKIP_UP=1" + mark_step "07.local_runtime_up" "skipped" + mark_step "08.connectivity" "skipped" + mark_step "09.ack_config" "skipped" + mark_step "10.teardown" "skipped" +fi