diff --git a/README.md b/README.md index 8bc25f9..c2cb86a 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,43 @@ curl -fsSL https://raw.githubusercontent.com/cloud-neutral-toolkit/observability > - logs endpoint: `/ingest/logs/insert` > - The script automatically verifies installation after setup. +### Optional: DeepFlow Agent on Client + +If you have deployed DeepFlow with `deepflow.yml`, you can install `deepflow-agent` on client nodes via the same script: + +```bash +# example: endpoint exposed by caddy grpc ingress (deepflow_grpc_domain:443) +curl -fsSL https://raw.githubusercontent.com/cloud-neutral-toolkit/observability.svc.plus/main/scripts/agent-install.sh \ + | bash -s -- \ + --endpoint https://observability.svc.plus/ingest/otlp \ + --deepflow-agent \ + --deepflow-grpc-endpoint deepflow-agent.svc.plus:443 \ + --deepflow-agent-download-url https://example.com/path/to/deepflow-agent +``` + +> If `deepflow-agent` binary already exists on host, replace `--deepflow-agent-download-url` with `--deepflow-agent-bin /path/to/deepflow-agent`. + +## 🚀 DeepFlow Deployment (Server Side) + +This repo now provides dedicated DeepFlow roles: + +- `deepflow_mysql` +- `deepflow_clickhouse_s3` +- `deepflow_server` + +Quick start: + +```bash +./configure -c app/deepflow +vi pigsty.yml # adjust domain/password/ports +./deploy.yml +./docker.yml +./deepflow.yml +./infra.yml -t caddy # apply deepflow_grpc_domain ingress +``` + +Default inventory template: `conf/app/deepflow.yml` + ### Remote client example (clawdbot.svc.plus) ```bash diff --git a/conf/app/deepflow.yml b/conf/app/deepflow.yml new file mode 100644 index 0000000..563e31d --- /dev/null +++ b/conf/app/deepflow.yml @@ -0,0 +1,92 @@ +--- +#==============================================================# +# File : deepflow.yml +# Desc : observability config for running DeepFlow stack +# Ctime : 2026-02-04 +# Mtime : 2026-02-04 +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +#==============================================================# + +# how to use this template: +# +# curl -fsSL https://repo.pigsty.io/get | bash; cd ~/pigsty +# ./bootstrap # prepare local repo & ansible +# ./configure -c app/deepflow # use this deepflow config template +# vi pigsty.yml # IMPORTANT: CHANGE CREDENTIALS / DOMAIN +# ./deploy.yml # install infra stack +# ./docker.yml # install docker & docker-compose +# ./deepflow.yml # install deepflow with three roles + +all: + children: + + deepflow: + hosts: { 10.10.10.10: {} } + vars: + deepflow_enabled: true + deepflow_mysql_enabled: true + deepflow_clickhouse_s3_enabled: true + + deepflow_data: /data/deepflow + + # role: deepflow_mysql + deepflow_mysql_port: 13306 + deepflow_mysql_root_password: DeepFlow.Root.ChangeMe + deepflow_mysql_user: deepflow + deepflow_mysql_password: DeepFlow.MySQL.ChangeMe + deepflow_mysql_database: deepflow + + # role: deepflow_clickhouse_s3 + deepflow_clickhouse_http_port: 18123 + deepflow_clickhouse_tcp_port: 19000 + deepflow_minio_api_port: 19090 + deepflow_minio_console_port: 19091 + deepflow_s3_bucket: deepflow + deepflow_s3_access_key: deepflow + deepflow_s3_secret_key: DeepFlow.S3.ChangeMe + deepflow_s3_region: us-east-1 + + # role: deepflow_server + deepflow_server_grpc_port: 20035 + deepflow_server_http_port: 20417 + deepflow_app_port: 20880 + deepflow_clickhouse_addr: host.docker.internal:19000 + deepflow_s3_endpoint: http://host.docker.internal:19090 + deepflow_mysql_addr: host.docker.internal:13306 + + infra: { hosts: { 10.10.10.10: { infra_seq: 1 } } } + etcd: { hosts: { 10.10.10.10: { etcd_seq: 1 } }, vars: { etcd_cluster: etcd } } + + vars: + version: v4.0.0 + admin_ip: 10.10.10.10 + region: default + node_tune: oltp + pg_conf: oltp.yml + docker_enabled: true + + # Caddy gRPC ingress for deepflow-agent: + caddy_enabled: true + deepflow_grpc_enabled: true + deepflow_grpc_domain: deepflow-agent.pigsty + deepflow_grpc_upstream: 127.0.0.1:20035 + + infra_portal: + home : { domain: svc.plus } + deepflow : { domain: deepflow.pigsty ,endpoint: "10.10.10.10:20880" } + + proxy_env: + no_proxy: "localhost,127.0.0.1,10.0.0.0/8,192.168.0.0/16,*.pigsty,*.aliyun.com,mirrors.*,*.tsinghua.edu.cn" + + repo_enabled: false + node_repo_modules: node,infra,pgsql + + grafana_admin_password: pigsty + grafana_view_password: DBUser.Viewer + pg_admin_password: DBUser.DBA + pg_monitor_password: DBUser.Monitor + pg_replication_password: DBUser.Replicator + patroni_password: Patroni.API + haproxy_admin_password: pigsty + minio_secret_key: S3User.MinIO + etcd_root_password: Etcd.Root diff --git a/deepflow.yml b/deepflow.yml new file mode 100644 index 0000000..80b88c8 --- /dev/null +++ b/deepflow.yml @@ -0,0 +1,26 @@ +#!/usr/bin/env ansible-playbook +--- +#==============================================================# +# File : deepflow.yml +# Desc : deploy deepflow stack with three dedicated roles +# Ctime : 2026-02-04 +# Mtime : 2026-02-04 +# Path : deepflow.yml +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +#==============================================================# + +- name: DEEPFLOW STACK + become: true + hosts: all + gather_facts: no + + roles: + - { role: node_id , tags: node-id, when: deepflow_enabled | default(true) | bool } + - { role: deepflow_mysql , tags: deepflow_mysql, when: deepflow_mysql_enabled | default(true) | bool } + - { role: deepflow_clickhouse_s3, tags: deepflow_clickhouse_s3, when: deepflow_clickhouse_s3_enabled | default(true) | bool } + - { role: deepflow_server , tags: deepflow_server, when: deepflow_enabled | default(true) | bool } + +# Usage: +# 1. Define deepflow group in pigsty.yml +# 2. Ensure docker is installed: ./docker.yml +# 3. Run ./deepflow.yml -l diff --git a/roles/deepflow_clickhouse_s3/README.md b/roles/deepflow_clickhouse_s3/README.md new file mode 100644 index 0000000..19e4f1a --- /dev/null +++ b/roles/deepflow_clickhouse_s3/README.md @@ -0,0 +1,10 @@ +# Role: deepflow_clickhouse_s3 + +Deploy ClickHouse + MinIO(S3) backend for DeepFlow with Docker Compose managed by systemd. + +## Key Variables + +- `deepflow_clickhouse_tcp_port` (default `19000`) +- `deepflow_clickhouse_http_port` (default `18123`) +- `deepflow_minio_api_port` (default `19090`) +- `deepflow_s3_access_key` / `deepflow_s3_secret_key` diff --git a/roles/deepflow_clickhouse_s3/defaults/main.yml b/roles/deepflow_clickhouse_s3/defaults/main.yml new file mode 100644 index 0000000..301a48c --- /dev/null +++ b/roles/deepflow_clickhouse_s3/defaults/main.yml @@ -0,0 +1,26 @@ +--- +#----------------------------------------------------------------- +# DEEPFLOW CLICKHOUSE + S3 +#----------------------------------------------------------------- +deepflow_clickhouse_s3_enabled: true + +deepflow_clickhouse_s3_stack_dir: /opt/deepflow-clickhouse-s3 +deepflow_clickhouse_s3_env_file: /etc/default/deepflow-clickhouse-s3 +deepflow_clickhouse_s3_compose_file: "{{ deepflow_clickhouse_s3_stack_dir }}/docker-compose.yml" + +deepflow_data: /data/deepflow +deepflow_clickhouse_data: "{{ deepflow_data }}/clickhouse" +deepflow_s3_data: "{{ deepflow_data }}/minio" + +deepflow_clickhouse_image: clickhouse/clickhouse-server:24.8 +deepflow_minio_image: minio/minio:RELEASE.2025-01-20T14-49-07Z + +deepflow_clickhouse_http_port: 18123 +deepflow_clickhouse_tcp_port: 19000 +deepflow_minio_api_port: 19090 +deepflow_minio_console_port: 19091 + +deepflow_s3_bucket: deepflow +deepflow_s3_access_key: deepflow +deepflow_s3_secret_key: DeepFlow.S3.ChangeMe +deepflow_s3_region: us-east-1 diff --git a/roles/deepflow_clickhouse_s3/meta/main.yml b/roles/deepflow_clickhouse_s3/meta/main.yml new file mode 100644 index 0000000..630fda5 --- /dev/null +++ b/roles/deepflow_clickhouse_s3/meta/main.yml @@ -0,0 +1,7 @@ +galaxy_info: + author: observability.svc.plus + description: Deploy DeepFlow ClickHouse + S3 backend via Docker Compose + license: Apache-2.0 + min_ansible_version: '2.10' + +dependencies: [] diff --git a/roles/deepflow_clickhouse_s3/tasks/main.yml b/roles/deepflow_clickhouse_s3/tasks/main.yml new file mode 100644 index 0000000..5a867c6 --- /dev/null +++ b/roles/deepflow_clickhouse_s3/tasks/main.yml @@ -0,0 +1,85 @@ +--- +#----------------------------------------------------------------# +# Preflight [deepflow_clickhouse_check] +#----------------------------------------------------------------# +- name: check deepflow clickhouse+s3 prerequisites + tags: deepflow_clickhouse_check + block: + - name: assert docker is enabled + assert: + that: + - docker_enabled is defined + - docker_enabled | bool + fail_msg: "docker_enabled=true is required for deepflow_clickhouse_s3 role" + + - name: check docker binary exists + command: docker --version + changed_when: false + +#----------------------------------------------------------------# +# Configure [deepflow_clickhouse_conf] +#----------------------------------------------------------------# +- name: configure deepflow clickhouse+s3 stack + tags: deepflow_clickhouse_conf + block: + - name: create deepflow clickhouse+s3 directories + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - "{{ deepflow_clickhouse_s3_stack_dir }}" + - "{{ deepflow_data }}" + - "{{ deepflow_clickhouse_data }}" + - "{{ deepflow_s3_data }}" + + - name: render deepflow clickhouse+s3 environment + template: + src: deepflow-clickhouse-s3.env.j2 + dest: "{{ deepflow_clickhouse_s3_env_file }}" + owner: root + group: root + mode: '0640' + + - name: render deepflow clickhouse+s3 docker compose + template: + src: docker-compose.yml.j2 + dest: "{{ deepflow_clickhouse_s3_compose_file }}" + owner: root + group: root + mode: '0644' + + - name: install deepflow clickhouse+s3 systemd unit + template: + src: deepflow-clickhouse-s3.svc + dest: "{{ systemd_dir }}/deepflow-clickhouse-s3.service" + owner: root + group: root + mode: '0644' + +#----------------------------------------------------------------# +# Launch [deepflow_clickhouse_up] +#----------------------------------------------------------------# +- name: launch deepflow clickhouse+s3 stack + tags: deepflow_clickhouse_up + block: + - name: start deepflow clickhouse+s3 service + systemd: + name: deepflow-clickhouse-s3 + state: restarted + enabled: yes + daemon_reload: yes + + - name: wait for deepflow clickhouse tcp port + wait_for: + host: 127.0.0.1 + port: "{{ deepflow_clickhouse_tcp_port }}" + timeout: 60 + + - name: wait for deepflow minio api port + wait_for: + host: 127.0.0.1 + port: "{{ deepflow_minio_api_port }}" + timeout: 60 diff --git a/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.env.j2 b/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.env.j2 new file mode 100644 index 0000000..a946bef --- /dev/null +++ b/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.env.j2 @@ -0,0 +1,15 @@ +DEEPFLOW_CLICKHOUSE_IMAGE={{ deepflow_clickhouse_image }} +DEEPFLOW_MINIO_IMAGE={{ deepflow_minio_image }} + +DEEPFLOW_CLICKHOUSE_DATA={{ deepflow_clickhouse_data }} +DEEPFLOW_S3_DATA={{ deepflow_s3_data }} + +DEEPFLOW_CLICKHOUSE_HTTP_PORT={{ deepflow_clickhouse_http_port }} +DEEPFLOW_CLICKHOUSE_TCP_PORT={{ deepflow_clickhouse_tcp_port }} +DEEPFLOW_MINIO_API_PORT={{ deepflow_minio_api_port }} +DEEPFLOW_MINIO_CONSOLE_PORT={{ deepflow_minio_console_port }} + +DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }} +DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }} +DEEPFLOW_S3_SECRET_KEY={{ deepflow_s3_secret_key }} +DEEPFLOW_S3_REGION={{ deepflow_s3_region }} diff --git a/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.svc b/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.svc new file mode 100644 index 0000000..201e07a --- /dev/null +++ b/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.svc @@ -0,0 +1,17 @@ +[Unit] +Description=DeepFlow ClickHouse + S3 (Docker Compose) +After=docker.service network-online.target +Wants=docker.service network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory={{ deepflow_clickhouse_s3_stack_dir }} +EnvironmentFile={{ deepflow_clickhouse_s3_env_file }} +ExecStart=/usr/bin/docker compose --env-file {{ deepflow_clickhouse_s3_env_file }} -f {{ deepflow_clickhouse_s3_compose_file }} up -d +ExecStop=/usr/bin/docker compose --env-file {{ deepflow_clickhouse_s3_env_file }} -f {{ deepflow_clickhouse_s3_compose_file }} down +TimeoutStartSec=0 +TimeoutStopSec=120 + +[Install] +WantedBy=multi-user.target diff --git a/roles/deepflow_clickhouse_s3/templates/docker-compose.yml.j2 b/roles/deepflow_clickhouse_s3/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..edd7d6b --- /dev/null +++ b/roles/deepflow_clickhouse_s3/templates/docker-compose.yml.j2 @@ -0,0 +1,33 @@ +version: '3.9' + +services: + clickhouse: + container_name: deepflow-clickhouse + image: ${DEEPFLOW_CLICKHOUSE_IMAGE} + restart: unless-stopped + environment: + CLICKHOUSE_DB: deepflow + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: '1' + ports: + - '${DEEPFLOW_CLICKHOUSE_HTTP_PORT}:8123' + - '${DEEPFLOW_CLICKHOUSE_TCP_PORT}:9000' + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - ${DEEPFLOW_CLICKHOUSE_DATA}:/var/lib/clickhouse + + minio: + container_name: deepflow-minio + image: ${DEEPFLOW_MINIO_IMAGE} + restart: unless-stopped + environment: + MINIO_ROOT_USER: ${DEEPFLOW_S3_ACCESS_KEY} + MINIO_ROOT_PASSWORD: ${DEEPFLOW_S3_SECRET_KEY} + command: server /data --console-address ':9001' + ports: + - '${DEEPFLOW_MINIO_API_PORT}:9000' + - '${DEEPFLOW_MINIO_CONSOLE_PORT}:9001' + volumes: + - ${DEEPFLOW_S3_DATA}:/data diff --git a/roles/deepflow_mysql/README.md b/roles/deepflow_mysql/README.md new file mode 100644 index 0000000..42ebe67 --- /dev/null +++ b/roles/deepflow_mysql/README.md @@ -0,0 +1,10 @@ +# Role: deepflow_mysql + +Deploy MySQL backend for DeepFlow with Docker Compose managed by systemd. + +## Key Variables + +- `deepflow_mysql_port` (default `13306`) +- `deepflow_mysql_database` (default `deepflow`) +- `deepflow_mysql_user` / `deepflow_mysql_password` +- `deepflow_mysql_root_password` diff --git a/roles/deepflow_mysql/defaults/main.yml b/roles/deepflow_mysql/defaults/main.yml new file mode 100644 index 0000000..37f8e12 --- /dev/null +++ b/roles/deepflow_mysql/defaults/main.yml @@ -0,0 +1,18 @@ +--- +#----------------------------------------------------------------- +# DEEPFLOW MYSQL +#----------------------------------------------------------------- +deepflow_mysql_enabled: true + +deepflow_mysql_stack_dir: /opt/deepflow-mysql +deepflow_mysql_env_file: /etc/default/deepflow-mysql +deepflow_mysql_compose_file: "{{ deepflow_mysql_stack_dir }}/docker-compose.yml" +deepflow_mysql_data: /data/deepflow/mysql + +deepflow_mysql_image: mysql:8.4 +deepflow_mysql_port: 13306 + +deepflow_mysql_root_password: DeepFlow.Root.ChangeMe +deepflow_mysql_user: deepflow +deepflow_mysql_password: DeepFlow.MySQL.ChangeMe +deepflow_mysql_database: deepflow diff --git a/roles/deepflow_mysql/meta/main.yml b/roles/deepflow_mysql/meta/main.yml new file mode 100644 index 0000000..6ffb6fa --- /dev/null +++ b/roles/deepflow_mysql/meta/main.yml @@ -0,0 +1,7 @@ +galaxy_info: + author: observability.svc.plus + description: Deploy DeepFlow MySQL backend via Docker Compose + license: Apache-2.0 + min_ansible_version: '2.10' + +dependencies: [] diff --git a/roles/deepflow_mysql/tasks/main.yml b/roles/deepflow_mysql/tasks/main.yml new file mode 100644 index 0000000..b1e4566 --- /dev/null +++ b/roles/deepflow_mysql/tasks/main.yml @@ -0,0 +1,77 @@ +--- +#--------------------------------------------------------------# +# Preflight [deepflow_mysql_check] +#--------------------------------------------------------------# +- name: check deepflow mysql prerequisites + tags: deepflow_mysql_check + block: + - name: assert docker is enabled + assert: + that: + - docker_enabled is defined + - docker_enabled | bool + fail_msg: "docker_enabled=true is required for deepflow_mysql role" + + - name: check docker binary exists + command: docker --version + changed_when: false + +#--------------------------------------------------------------# +# Configure [deepflow_mysql_conf] +#--------------------------------------------------------------# +- name: configure deepflow mysql stack + tags: deepflow_mysql_conf + block: + - name: create deepflow mysql directories + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - "{{ deepflow_mysql_stack_dir }}" + - "{{ deepflow_mysql_data }}" + + - name: render deepflow mysql environment + template: + src: deepflow-mysql.env.j2 + dest: "{{ deepflow_mysql_env_file }}" + owner: root + group: root + mode: '0640' + + - name: render deepflow mysql docker compose + template: + src: docker-compose.yml.j2 + dest: "{{ deepflow_mysql_compose_file }}" + owner: root + group: root + mode: '0644' + + - name: install deepflow mysql systemd unit + template: + src: deepflow-mysql.svc + dest: "{{ systemd_dir }}/deepflow-mysql.service" + owner: root + group: root + mode: '0644' + +#--------------------------------------------------------------# +# Launch [deepflow_mysql_up] +#--------------------------------------------------------------# +- name: launch deepflow mysql stack + tags: deepflow_mysql_up + block: + - name: start deepflow mysql service + systemd: + name: deepflow-mysql + state: restarted + enabled: yes + daemon_reload: yes + + - name: wait for deepflow mysql port + wait_for: + host: 127.0.0.1 + port: "{{ deepflow_mysql_port }}" + timeout: 60 diff --git a/roles/deepflow_mysql/templates/deepflow-mysql.env.j2 b/roles/deepflow_mysql/templates/deepflow-mysql.env.j2 new file mode 100644 index 0000000..26c0a42 --- /dev/null +++ b/roles/deepflow_mysql/templates/deepflow-mysql.env.j2 @@ -0,0 +1,8 @@ +DEEPFLOW_MYSQL_IMAGE={{ deepflow_mysql_image }} +DEEPFLOW_MYSQL_PORT={{ deepflow_mysql_port }} +DEEPFLOW_MYSQL_DATA={{ deepflow_mysql_data }} + +DEEPFLOW_MYSQL_ROOT_PASSWORD={{ deepflow_mysql_root_password }} +DEEPFLOW_MYSQL_USER={{ deepflow_mysql_user }} +DEEPFLOW_MYSQL_PASSWORD={{ deepflow_mysql_password }} +DEEPFLOW_MYSQL_DATABASE={{ deepflow_mysql_database }} diff --git a/roles/deepflow_mysql/templates/deepflow-mysql.svc b/roles/deepflow_mysql/templates/deepflow-mysql.svc new file mode 100644 index 0000000..d716712 --- /dev/null +++ b/roles/deepflow_mysql/templates/deepflow-mysql.svc @@ -0,0 +1,17 @@ +[Unit] +Description=DeepFlow MySQL (Docker Compose) +After=docker.service network-online.target +Wants=docker.service network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory={{ deepflow_mysql_stack_dir }} +EnvironmentFile={{ deepflow_mysql_env_file }} +ExecStart=/usr/bin/docker compose --env-file {{ deepflow_mysql_env_file }} -f {{ deepflow_mysql_compose_file }} up -d +ExecStop=/usr/bin/docker compose --env-file {{ deepflow_mysql_env_file }} -f {{ deepflow_mysql_compose_file }} down +TimeoutStartSec=0 +TimeoutStopSec=120 + +[Install] +WantedBy=multi-user.target diff --git a/roles/deepflow_mysql/templates/docker-compose.yml.j2 b/roles/deepflow_mysql/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..2429f23 --- /dev/null +++ b/roles/deepflow_mysql/templates/docker-compose.yml.j2 @@ -0,0 +1,16 @@ +version: '3.9' + +services: + mysql: + container_name: deepflow-mysql + image: ${DEEPFLOW_MYSQL_IMAGE} + restart: unless-stopped + environment: + MYSQL_ROOT_PASSWORD: ${DEEPFLOW_MYSQL_ROOT_PASSWORD} + MYSQL_DATABASE: ${DEEPFLOW_MYSQL_DATABASE} + MYSQL_USER: ${DEEPFLOW_MYSQL_USER} + MYSQL_PASSWORD: ${DEEPFLOW_MYSQL_PASSWORD} + ports: + - '${DEEPFLOW_MYSQL_PORT}:3306' + volumes: + - ${DEEPFLOW_MYSQL_DATA}:/var/lib/mysql diff --git a/roles/deepflow_server/README.md b/roles/deepflow_server/README.md new file mode 100644 index 0000000..0a2f2a0 --- /dev/null +++ b/roles/deepflow_server/README.md @@ -0,0 +1,21 @@ +# Role: deepflow_server + +Deploy DeepFlow control plane (`deepflow-server` + `deepflow-app`) with Docker Compose managed by systemd. + +This role expects backend dependencies from separate roles: + +- `deepflow_mysql` +- `deepflow_clickhouse_s3` + +## Usage + +1. Ensure Docker is installed (`./docker.yml`) and `docker_enabled: true`. +2. Run backend roles first, then this role (see `deepflow.yml`). + +## Key Variables + +- `deepflow_server_grpc_port` (default `20035`) +- `deepflow_server_http_port` (default `20417`) +- `deepflow_app_port` (default `20880`) +- `deepflow_clickhouse_addr` (default `host.docker.internal:19000`) +- `deepflow_s3_endpoint` (default `http://host.docker.internal:19090`) diff --git a/roles/deepflow_server/defaults/main.yml b/roles/deepflow_server/defaults/main.yml new file mode 100644 index 0000000..d70832a --- /dev/null +++ b/roles/deepflow_server/defaults/main.yml @@ -0,0 +1,32 @@ +--- +#----------------------------------------------------------------- +# DEEPFLOW SERVER +#----------------------------------------------------------------- +deepflow_enabled: true + +deepflow_stack_dir: /opt/deepflow-server +deepflow_data: /data/deepflow +deepflow_env_file: /etc/default/deepflow-server +deepflow_compose_file: "{{ deepflow_stack_dir }}/docker-compose.yml" + +# images (pin to specific tags before production) +deepflow_server_image: deepflowio/deepflow-server-ce:latest +deepflow_app_image: deepflowio/deepflow-app-ce:latest + +# exposed ports +deepflow_server_grpc_port: 20035 +deepflow_server_http_port: 20417 +deepflow_app_port: 20880 + +# backend endpoints (provided by dedicated roles) +deepflow_clickhouse_addr: host.docker.internal:19000 +deepflow_s3_endpoint: http://host.docker.internal:19090 +deepflow_s3_bucket: deepflow +deepflow_s3_access_key: deepflow +deepflow_s3_secret_key: DeepFlow.S3.ChangeMe +deepflow_s3_region: us-east-1 + +deepflow_mysql_addr: host.docker.internal:13306 +deepflow_mysql_user: deepflow +deepflow_mysql_password: DeepFlow.MySQL.ChangeMe +deepflow_mysql_database: deepflow diff --git a/roles/deepflow_server/meta/main.yml b/roles/deepflow_server/meta/main.yml new file mode 100644 index 0000000..e817a10 --- /dev/null +++ b/roles/deepflow_server/meta/main.yml @@ -0,0 +1,7 @@ +galaxy_info: + author: observability.svc.plus + description: Deploy DeepFlow stack via Docker Compose + license: Apache-2.0 + min_ansible_version: '2.10' + +dependencies: [] diff --git a/roles/deepflow_server/tasks/main.yml b/roles/deepflow_server/tasks/main.yml new file mode 100644 index 0000000..3de7620 --- /dev/null +++ b/roles/deepflow_server/tasks/main.yml @@ -0,0 +1,84 @@ +--- +#--------------------------------------------------------------# +# Preflight [deepflow_check] +#--------------------------------------------------------------# +- name: check deepflow prerequisites + tags: deepflow_check + block: + - name: assert docker is enabled + assert: + that: + - docker_enabled is defined + - docker_enabled | bool + fail_msg: "docker_enabled=true is required for deepflow_server role" + + - name: check docker binary exists + command: docker --version + changed_when: false + +#--------------------------------------------------------------# +# Configure [deepflow_conf] +#--------------------------------------------------------------# +- name: configure deepflow-server stack + tags: deepflow_conf + block: + - name: create deepflow directories + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - "{{ deepflow_stack_dir }}" + - "{{ deepflow_data }}" + - "{{ deepflow_data }}/server" + + - name: render deepflow environment + template: + src: deepflow.env.j2 + dest: "{{ deepflow_env_file }}" + owner: root + group: root + mode: '0640' + + - name: render deepflow docker compose + template: + src: docker-compose.yml.j2 + dest: "{{ deepflow_compose_file }}" + owner: root + group: root + mode: '0644' + + - name: install deepflow systemd unit + template: + src: deepflow-server.svc + dest: "{{ systemd_dir }}/deepflow-server.service" + owner: root + group: root + mode: '0644' + +#--------------------------------------------------------------# +# Launch [deepflow_up] +#--------------------------------------------------------------# +- name: launch deepflow-server stack + tags: deepflow_up + block: + - name: start deepflow-server service + systemd: + name: deepflow-server + state: restarted + enabled: yes + daemon_reload: yes + + - name: wait for deepflow gRPC port + wait_for: + host: 127.0.0.1 + port: "{{ deepflow_server_grpc_port }}" + timeout: 60 + + - name: wait for deepflow app port + wait_for: + host: 127.0.0.1 + port: "{{ deepflow_app_port }}" + timeout: 60 diff --git a/roles/deepflow_server/templates/deepflow-server.svc b/roles/deepflow_server/templates/deepflow-server.svc new file mode 100644 index 0000000..179f993 --- /dev/null +++ b/roles/deepflow_server/templates/deepflow-server.svc @@ -0,0 +1,17 @@ +[Unit] +Description=DeepFlow Server Stack (Docker Compose) +After=docker.service network-online.target +Wants=docker.service network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory={{ deepflow_stack_dir }} +EnvironmentFile={{ deepflow_env_file }} +ExecStart=/usr/bin/docker compose --env-file {{ deepflow_env_file }} -f {{ deepflow_compose_file }} up -d +ExecStop=/usr/bin/docker compose --env-file {{ deepflow_env_file }} -f {{ deepflow_compose_file }} down +TimeoutStartSec=0 +TimeoutStopSec=120 + +[Install] +WantedBy=multi-user.target diff --git a/roles/deepflow_server/templates/deepflow.env.j2 b/roles/deepflow_server/templates/deepflow.env.j2 new file mode 100644 index 0000000..943ff42 --- /dev/null +++ b/roles/deepflow_server/templates/deepflow.env.j2 @@ -0,0 +1,21 @@ +DEEPFLOW_STACK_DIR={{ deepflow_stack_dir }} +DEEPFLOW_DATA={{ deepflow_data }} + +DEEPFLOW_SERVER_IMAGE={{ deepflow_server_image }} +DEEPFLOW_APP_IMAGE={{ deepflow_app_image }} + +DEEPFLOW_SERVER_GRPC_PORT={{ deepflow_server_grpc_port }} +DEEPFLOW_SERVER_HTTP_PORT={{ deepflow_server_http_port }} +DEEPFLOW_APP_PORT={{ deepflow_app_port }} + +DEEPFLOW_CLICKHOUSE_ADDR={{ deepflow_clickhouse_addr }} +DEEPFLOW_S3_ENDPOINT={{ deepflow_s3_endpoint }} +DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }} +DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }} +DEEPFLOW_S3_SECRET_KEY={{ deepflow_s3_secret_key }} +DEEPFLOW_S3_REGION={{ deepflow_s3_region }} + +DEEPFLOW_MYSQL_ADDR={{ deepflow_mysql_addr }} +DEEPFLOW_MYSQL_USER={{ deepflow_mysql_user }} +DEEPFLOW_MYSQL_PASSWORD={{ deepflow_mysql_password }} +DEEPFLOW_MYSQL_DATABASE={{ deepflow_mysql_database }} diff --git a/roles/deepflow_server/templates/docker-compose.yml.j2 b/roles/deepflow_server/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..861f1a6 --- /dev/null +++ b/roles/deepflow_server/templates/docker-compose.yml.j2 @@ -0,0 +1,36 @@ +version: '3.9' + +services: + deepflow-server: + container_name: deepflow-server + image: ${DEEPFLOW_SERVER_IMAGE} + restart: unless-stopped + extra_hosts: + - 'host.docker.internal:host-gateway' + environment: + DEEPFLOW_CLICKHOUSE_ADDR: ${DEEPFLOW_CLICKHOUSE_ADDR} + DEEPFLOW_S3_ENDPOINT: ${DEEPFLOW_S3_ENDPOINT} + DEEPFLOW_S3_BUCKET: ${DEEPFLOW_S3_BUCKET} + DEEPFLOW_S3_ACCESS_KEY: ${DEEPFLOW_S3_ACCESS_KEY} + DEEPFLOW_S3_SECRET_KEY: ${DEEPFLOW_S3_SECRET_KEY} + DEEPFLOW_S3_REGION: ${DEEPFLOW_S3_REGION} + DEEPFLOW_MYSQL_ADDR: ${DEEPFLOW_MYSQL_ADDR} + DEEPFLOW_MYSQL_USER: ${DEEPFLOW_MYSQL_USER} + DEEPFLOW_MYSQL_PASSWORD: ${DEEPFLOW_MYSQL_PASSWORD} + DEEPFLOW_MYSQL_DATABASE: ${DEEPFLOW_MYSQL_DATABASE} + ports: + - '${DEEPFLOW_SERVER_GRPC_PORT}:20035' + - '${DEEPFLOW_SERVER_HTTP_PORT}:20417' + volumes: + - ${DEEPFLOW_DATA}/server:/var/lib/deepflow + + deepflow-app: + container_name: deepflow-app + image: ${DEEPFLOW_APP_IMAGE} + restart: unless-stopped + depends_on: + - deepflow-server + environment: + DEEPFLOW_SERVER_ADDR: http://deepflow-server:20417 + ports: + - '${DEEPFLOW_APP_PORT}:80' diff --git a/roles/infra/defaults/main.yml b/roles/infra/defaults/main.yml index c53b5b3..7401882 100644 --- a/roles/infra/defaults/main.yml +++ b/roles/infra/defaults/main.yml @@ -38,6 +38,10 @@ infra_extra_services: [] # extra services to be added on infra home pag # CADDY vs NGINX #----------------------------------------------------------------- caddy_enabled: true # use caddy as ingress controller? +# Optional DeepFlow agent ingress: TLS gRPC -> Caddy -> deepflow-server(h2c) +deepflow_grpc_enabled: false # expose dedicated gRPC ingress for deepflow-agent? +deepflow_grpc_domain: deepflow-agent.svc.plus +deepflow_grpc_upstream: 127.0.0.1:20035 nginx_enabled: false # enable nginx on this infra node? nginx_clean: false # clean existing nginx config during init? nginx_exporter_enabled: true # enable nginx_exporter on this infra node? diff --git a/roles/infra/templates/caddy/Caddyfile b/roles/infra/templates/caddy/Caddyfile index 29d58fe..d75334b 100644 --- a/roles/infra/templates/caddy/Caddyfile +++ b/roles/infra/templates/caddy/Caddyfile @@ -150,3 +150,14 @@ file_server } } + +{% if deepflow_grpc_enabled | default(false) %} +{{ deepflow_grpc_domain | default('deepflow-agent.svc.plus') }} { + # TLS terminates at Caddy, then proxy h2c gRPC to deepflow-server. + reverse_proxy {{ deepflow_grpc_upstream | default('127.0.0.1:20035') }} { + transport http { + versions h2c + } + } +} +{% endif %} diff --git a/scripts/agent-install.sh b/scripts/agent-install.sh index 67f7b57..ead4f70 100755 --- a/scripts/agent-install.sh +++ b/scripts/agent-install.sh @@ -17,6 +17,11 @@ METRICS_ENDPOINT="" LOGS_ENDPOINT="" METRICS_ENDPOINT_SET=false LOGS_ENDPOINT_SET=false +DEEPFLOW_AGENT_ENABLED=false +DEEPFLOW_GRPC_ENDPOINT="" +DEEPFLOW_AGENT_ENDPOINT_ARG="--grpc-server" +DEEPFLOW_AGENT_DOWNLOAD_URL="" +DEEPFLOW_AGENT_BIN="${BIN_DIR}/deepflow-agent" AUTO_YES=false GREEN='\033[0;32m' @@ -46,6 +51,11 @@ Options: --endpoint URL Base ingest endpoint (default: ${DEFAULT_ENDPOINT}) --metrics-endpoint URL Prometheus remote_write endpoint (optional override) --logs-endpoint URL Loki push endpoint (optional override) + --deepflow-agent Install and enable deepflow-agent service + --deepflow-grpc-endpoint HOST:PORT DeepFlow gRPC endpoint (required if --deepflow-agent) + --deepflow-agent-endpoint-arg ARG deepflow-agent endpoint arg (default: ${DEEPFLOW_AGENT_ENDPOINT_ARG}) + --deepflow-agent-download-url URL Download deepflow-agent binary to ${DEEPFLOW_AGENT_BIN} + --deepflow-agent-bin PATH Use existing deepflow-agent binary path -y, --yes Non-interactive mode -h, --help Show help @@ -101,6 +111,42 @@ while [[ $# -gt 0 ]]; do LOGS_ENDPOINT_SET=true shift ;; + --deepflow-agent) + DEEPFLOW_AGENT_ENABLED=true + shift + ;; + --deepflow-grpc-endpoint) + DEEPFLOW_GRPC_ENDPOINT="$2" + shift 2 + ;; + --deepflow-grpc-endpoint=*) + DEEPFLOW_GRPC_ENDPOINT="${1#*=}" + shift + ;; + --deepflow-agent-endpoint-arg) + DEEPFLOW_AGENT_ENDPOINT_ARG="$2" + shift 2 + ;; + --deepflow-agent-endpoint-arg=*) + DEEPFLOW_AGENT_ENDPOINT_ARG="${1#*=}" + shift + ;; + --deepflow-agent-download-url) + DEEPFLOW_AGENT_DOWNLOAD_URL="$2" + shift 2 + ;; + --deepflow-agent-download-url=*) + DEEPFLOW_AGENT_DOWNLOAD_URL="${1#*=}" + shift + ;; + --deepflow-agent-bin) + DEEPFLOW_AGENT_BIN="$2" + shift 2 + ;; + --deepflow-agent-bin=*) + DEEPFLOW_AGENT_BIN="${1#*=}" + shift + ;; -y|--yes) AUTO_YES=true shift @@ -127,6 +173,9 @@ fi if [[ -z "${LOGS_ENDPOINT}" ]]; then LOGS_ENDPOINT="${base_endpoint}/ingest/logs/insert" fi +if [[ "${DEEPFLOW_AGENT_ENABLED}" == "true" && -z "${DEEPFLOW_GRPC_ENDPOINT}" ]]; then + DEEPFLOW_GRPC_ENDPOINT="deepflow-agent.${base_endpoint#*://}:443" +fi # observability server should bypass external HTTPS ingress for local self-monitoring local_host="$(hostname -f 2>/dev/null || hostname)" @@ -382,17 +431,76 @@ WantedBy=multi-user.target" systemctl restart vector } +install_deepflow_agent() { + if [[ "${DEEPFLOW_AGENT_ENABLED}" != "true" ]]; then + return 0 + fi + + if [[ -z "${DEEPFLOW_GRPC_ENDPOINT}" ]]; then + log_error "DeepFlow agent enabled but --deepflow-grpc-endpoint is empty." + exit 1 + fi + + if [[ -n "${DEEPFLOW_AGENT_DOWNLOAD_URL}" ]]; then + log_info "Downloading deepflow-agent binary..." + curl -fL --progress-bar "${DEEPFLOW_AGENT_DOWNLOAD_URL}" -o "${DEEPFLOW_AGENT_BIN}" + chmod 0755 "${DEEPFLOW_AGENT_BIN}" + elif [[ ! -x "${DEEPFLOW_AGENT_BIN}" ]]; then + if command -v deepflow-agent >/dev/null 2>&1; then + DEEPFLOW_AGENT_BIN="$(command -v deepflow-agent)" + else + log_error "deepflow-agent binary not found. Use --deepflow-agent-download-url or --deepflow-agent-bin." + exit 1 + fi + fi + + cat < "${CONFIG_DIR}/deepflow-agent.env" +DEEPFLOW_GRPC_ENDPOINT=${DEEPFLOW_GRPC_ENDPOINT} +DEEPFLOW_AGENT_ENDPOINT_ARG=${DEEPFLOW_AGENT_ENDPOINT_ARG} +EOF + + cat <<'EOF' > "${BIN_DIR}/run-deepflow-agent.sh" +#!/bin/bash +set -euo pipefail +: "${DEEPFLOW_AGENT_BIN:?missing DEEPFLOW_AGENT_BIN}" +: "${DEEPFLOW_AGENT_ENDPOINT_ARG:?missing DEEPFLOW_AGENT_ENDPOINT_ARG}" +: "${DEEPFLOW_GRPC_ENDPOINT:?missing DEEPFLOW_GRPC_ENDPOINT}" +exec "${DEEPFLOW_AGENT_BIN}" "${DEEPFLOW_AGENT_ENDPOINT_ARG}" "${DEEPFLOW_GRPC_ENDPOINT}" +EOF + chmod 0755 "${BIN_DIR}/run-deepflow-agent.sh" + + write_unit_if_changed "deepflow_agent" "[Unit] +Description=DeepFlow Agent +After=network-online.target +Wants=network-online.target + +[Service] +User=root +EnvironmentFile=${CONFIG_DIR}/deepflow-agent.env +Environment=DEEPFLOW_AGENT_BIN=${DEEPFLOW_AGENT_BIN} +ExecStart=${BIN_DIR}/run-deepflow-agent.sh +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target" + + systemctl enable --now deepflow_agent + systemctl restart deepflow_agent +} + uninstall_agent() { confirm "This will uninstall observability agent components. Continue?" || { log_info "Cancelled." return 0 } - for svc in vector process_exporter node_exporter; do + for svc in deepflow_agent vector process_exporter node_exporter; do systemctl disable --now "${svc}" >/dev/null 2>&1 || true rm -f "/etc/systemd/system/${svc}.service" done systemctl daemon-reload + rm -f "${BIN_DIR}/run-deepflow-agent.sh" "${CONFIG_DIR}/deepflow-agent.env" rm -rf "${INSTALL_DIR}" log_success "Agent components uninstalled." } @@ -408,6 +516,14 @@ verify_installation() { systemctl status "${service}" --no-pager | head -n 20 || true fi done + if [[ "${DEEPFLOW_AGENT_ENABLED}" == "true" ]]; then + if systemctl is-active --quiet deepflow_agent; then + log_success "Service 'deepflow_agent' is running" + else + log_fail "Service 'deepflow_agent' is NOT running" + systemctl status deepflow_agent --no-pager | head -n 20 || true + fi + fi log_info "Checking ports..." for item in "9100 Node Exporter" "9256 Process Exporter"; do @@ -435,9 +551,13 @@ deploy_agent() { log_info "Base endpoint=${ENDPOINT}" log_info "Metrics endpoint=${METRICS_ENDPOINT}" log_info "Logs endpoint=${LOGS_ENDPOINT}" + if [[ "${DEEPFLOW_AGENT_ENABLED}" == "true" ]]; then + log_info "DeepFlow endpoint=${DEEPFLOW_GRPC_ENDPOINT}" + fi install_node_exporter install_process_exporter install_vector + install_deepflow_agent verify_installation log_success "Agent deploy/upgrade complete." print_endpoint_summary