ai-workspace-infra/observability.svc.plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2 from cloud-neutral-toolkit/codex/deepflow-server-role

Browse Source 
DeepFlow server roles + optional client deepflow-agent
This commit is contained in:
Haitao Pan 2026-02-09 16:44:21 +08:00 committed by GitHub
commit df5308d900
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 855 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
README.md
View File

@ -54,6 +54,43 @@ curl -fsSL https://raw.githubusercontent.com/cloud-neutral-toolkit/observability
> - logs endpoint: `/ingest/logs/insert`
> - The script automatically verifies installation after setup.
### Optional: DeepFlow Agent on Client
If you have deployed DeepFlow with `deepflow.yml`, you can install `deepflow-agent` on client nodes via the same script:
```bash
# example: endpoint exposed by caddy grpc ingress (deepflow_grpc_domain:443)
curl -fsSL https://raw.githubusercontent.com/cloud-neutral-toolkit/observability.svc.plus/main/scripts/agent-install.sh \
| bash -s -- \
--endpoint https://observability.svc.plus/ingest/otlp \
--deepflow-agent \
--deepflow-grpc-endpoint deepflow-agent.svc.plus:443 \
--deepflow-agent-download-url https://example.com/path/to/deepflow-agent
```
> If `deepflow-agent` binary already exists on host, replace `--deepflow-agent-download-url` with `--deepflow-agent-bin /path/to/deepflow-agent`.
## 🚀 DeepFlow Deployment (Server Side)
This repo now provides dedicated DeepFlow roles:
- `deepflow_mysql`
- `deepflow_clickhouse_s3`
- `deepflow_server`
Quick start:
```bash
./configure -c app/deepflow
vi pigsty.yml # adjust domain/password/ports
./deploy.yml
./docker.yml
./deepflow.yml
./infra.yml -t caddy # apply deepflow_grpc_domain ingress
```
Default inventory template: `conf/app/deepflow.yml`
### Remote client example (clawdbot.svc.plus)
```bash

92
conf/app/deepflow.yml Normal file
View File

@ -0,0 +1,92 @@
---
#==============================================================#
# File : deepflow.yml
# Desc : observability config for running DeepFlow stack
# Ctime : 2026-02-04
# Mtime : 2026-02-04
# License : Apache-2.0 @ https://pigsty.io/docs/about/license/
#==============================================================#
# how to use this template:
#
# curl -fsSL https://repo.pigsty.io/get | bash; cd ~/pigsty
# ./bootstrap # prepare local repo & ansible
# ./configure -c app/deepflow # use this deepflow config template
# vi pigsty.yml # IMPORTANT: CHANGE CREDENTIALS / DOMAIN
# ./deploy.yml # install infra stack
# ./docker.yml # install docker & docker-compose
# ./deepflow.yml # install deepflow with three roles
all:
children:
deepflow:
hosts: { 10.10.10.10: {} }
vars:
deepflow_enabled: true
deepflow_mysql_enabled: true
deepflow_clickhouse_s3_enabled: true
deepflow_data: /data/deepflow
# role: deepflow_mysql
deepflow_mysql_port: 13306
deepflow_mysql_root_password: DeepFlow.Root.ChangeMe
deepflow_mysql_user: deepflow
deepflow_mysql_password: DeepFlow.MySQL.ChangeMe
deepflow_mysql_database: deepflow
# role: deepflow_clickhouse_s3
deepflow_clickhouse_http_port: 18123
deepflow_clickhouse_tcp_port: 19000
deepflow_minio_api_port: 19090
deepflow_minio_console_port: 19091
deepflow_s3_bucket: deepflow
deepflow_s3_access_key: deepflow
deepflow_s3_secret_key: DeepFlow.S3.ChangeMe
deepflow_s3_region: us-east-1
# role: deepflow_server
deepflow_server_grpc_port: 20035
deepflow_server_http_port: 20417
deepflow_app_port: 20880
deepflow_clickhouse_addr: host.docker.internal:19000
deepflow_s3_endpoint: http://host.docker.internal:19090
deepflow_mysql_addr: host.docker.internal:13306
infra: { hosts: { 10.10.10.10: { infra_seq: 1 } } }
etcd: { hosts: { 10.10.10.10: { etcd_seq: 1 } }, vars: { etcd_cluster: etcd } }
vars:
version: v4.0.0
admin_ip: 10.10.10.10
region: default
node_tune: oltp
pg_conf: oltp.yml
docker_enabled: true
# Caddy gRPC ingress for deepflow-agent:
caddy_enabled: true
deepflow_grpc_enabled: true
deepflow_grpc_domain: deepflow-agent.pigsty
deepflow_grpc_upstream: 127.0.0.1:20035
infra_portal:
home : { domain: svc.plus }
deepflow : { domain: deepflow.pigsty ,endpoint: "10.10.10.10:20880" }
proxy_env:
no_proxy: "localhost,127.0.0.1,10.0.0.0/8,192.168.0.0/16,*.pigsty,*.aliyun.com,mirrors.*,*.tsinghua.edu.cn"
repo_enabled: false
node_repo_modules: node,infra,pgsql
grafana_admin_password: pigsty
grafana_view_password: DBUser.Viewer
pg_admin_password: DBUser.DBA
pg_monitor_password: DBUser.Monitor
pg_replication_password: DBUser.Replicator
patroni_password: Patroni.API
haproxy_admin_password: pigsty
minio_secret_key: S3User.MinIO
etcd_root_password: Etcd.Root

26
deepflow.yml Normal file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env ansible-playbook
---
#==============================================================#
# File : deepflow.yml
# Desc : deploy deepflow stack with three dedicated roles
# Ctime : 2026-02-04
# Mtime : 2026-02-04
# Path : deepflow.yml
# License : Apache-2.0 @ https://pigsty.io/docs/about/license/
#==============================================================#
- name: DEEPFLOW STACK
become: true
hosts: all
gather_facts: no
roles:
- { role: node_id , tags: node-id, when: deepflow_enabled | default(true) | bool }
- { role: deepflow_mysql , tags: deepflow_mysql, when: deepflow_mysql_enabled | default(true) | bool }
- { role: deepflow_clickhouse_s3, tags: deepflow_clickhouse_s3, when: deepflow_clickhouse_s3_enabled | default(true) | bool }
- { role: deepflow_server , tags: deepflow_server, when: deepflow_enabled | default(true) | bool }
# Usage:
# 1. Define deepflow group in pigsty.yml
# 2. Ensure docker is installed: ./docker.yml
# 3. Run ./deepflow.yml -l <deepflow_group>

10
roles/deepflow_clickhouse_s3/README.md Normal file
View File

@ -0,0 +1,10 @@
# Role: deepflow_clickhouse_s3
Deploy ClickHouse + MinIO(S3) backend for DeepFlow with Docker Compose managed by systemd.
## Key Variables
- `deepflow_clickhouse_tcp_port` (default `19000`)
- `deepflow_clickhouse_http_port` (default `18123`)
- `deepflow_minio_api_port` (default `19090`)
- `deepflow_s3_access_key` / `deepflow_s3_secret_key`

26
roles/deepflow_clickhouse_s3/defaults/main.yml Normal file
View File

@ -0,0 +1,26 @@
---
#-----------------------------------------------------------------
# DEEPFLOW CLICKHOUSE + S3
#-----------------------------------------------------------------
deepflow_clickhouse_s3_enabled: true
deepflow_clickhouse_s3_stack_dir: /opt/deepflow-clickhouse-s3
deepflow_clickhouse_s3_env_file: /etc/default/deepflow-clickhouse-s3
deepflow_clickhouse_s3_compose_file: "{{ deepflow_clickhouse_s3_stack_dir }}/docker-compose.yml"
deepflow_data: /data/deepflow
deepflow_clickhouse_data: "{{ deepflow_data }}/clickhouse"
deepflow_s3_data: "{{ deepflow_data }}/minio"
deepflow_clickhouse_image: clickhouse/clickhouse-server:24.8
deepflow_minio_image: minio/minio:RELEASE.2025-01-20T14-49-07Z
deepflow_clickhouse_http_port: 18123
deepflow_clickhouse_tcp_port: 19000
deepflow_minio_api_port: 19090
deepflow_minio_console_port: 19091
deepflow_s3_bucket: deepflow
deepflow_s3_access_key: deepflow
deepflow_s3_secret_key: DeepFlow.S3.ChangeMe
deepflow_s3_region: us-east-1

7
roles/deepflow_clickhouse_s3/meta/main.yml Normal file
View File

@ -0,0 +1,7 @@
galaxy_info:
author: observability.svc.plus
description: Deploy DeepFlow ClickHouse + S3 backend via Docker Compose
license: Apache-2.0
min_ansible_version: '2.10'
dependencies: []

85
roles/deepflow_clickhouse_s3/tasks/main.yml Normal file
View File

@ -0,0 +1,85 @@
---
#----------------------------------------------------------------#
# Preflight [deepflow_clickhouse_check]
#----------------------------------------------------------------#
- name: check deepflow clickhouse+s3 prerequisites
tags: deepflow_clickhouse_check
block:
- name: assert docker is enabled
assert:
that:
- docker_enabled is defined
- docker_enabled | bool
fail_msg: "docker_enabled=true is required for deepflow_clickhouse_s3 role"
- name: check docker binary exists
command: docker --version
changed_when: false
#----------------------------------------------------------------#
# Configure [deepflow_clickhouse_conf]
#----------------------------------------------------------------#
- name: configure deepflow clickhouse+s3 stack
tags: deepflow_clickhouse_conf
block:
- name: create deepflow clickhouse+s3 directories
file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: '0755'
loop:
- "{{ deepflow_clickhouse_s3_stack_dir }}"
- "{{ deepflow_data }}"
- "{{ deepflow_clickhouse_data }}"
- "{{ deepflow_s3_data }}"
- name: render deepflow clickhouse+s3 environment
template:
src: deepflow-clickhouse-s3.env.j2
dest: "{{ deepflow_clickhouse_s3_env_file }}"
owner: root
group: root
mode: '0640'
- name: render deepflow clickhouse+s3 docker compose
template:
src: docker-compose.yml.j2
dest: "{{ deepflow_clickhouse_s3_compose_file }}"
owner: root
group: root
mode: '0644'
- name: install deepflow clickhouse+s3 systemd unit
template:
src: deepflow-clickhouse-s3.svc
dest: "{{ systemd_dir }}/deepflow-clickhouse-s3.service"
owner: root
group: root
mode: '0644'
#----------------------------------------------------------------#
# Launch [deepflow_clickhouse_up]
#----------------------------------------------------------------#
- name: launch deepflow clickhouse+s3 stack
tags: deepflow_clickhouse_up
block:
- name: start deepflow clickhouse+s3 service
systemd:
name: deepflow-clickhouse-s3
state: restarted
enabled: yes
daemon_reload: yes
- name: wait for deepflow clickhouse tcp port
wait_for:
host: 127.0.0.1
port: "{{ deepflow_clickhouse_tcp_port }}"
timeout: 60
- name: wait for deepflow minio api port
wait_for:
host: 127.0.0.1
port: "{{ deepflow_minio_api_port }}"
timeout: 60

15
roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.env.j2 Normal file
View File

@ -0,0 +1,15 @@
DEEPFLOW_CLICKHOUSE_IMAGE={{ deepflow_clickhouse_image }}
DEEPFLOW_MINIO_IMAGE={{ deepflow_minio_image }}
DEEPFLOW_CLICKHOUSE_DATA={{ deepflow_clickhouse_data }}
DEEPFLOW_S3_DATA={{ deepflow_s3_data }}
DEEPFLOW_CLICKHOUSE_HTTP_PORT={{ deepflow_clickhouse_http_port }}
DEEPFLOW_CLICKHOUSE_TCP_PORT={{ deepflow_clickhouse_tcp_port }}
DEEPFLOW_MINIO_API_PORT={{ deepflow_minio_api_port }}
DEEPFLOW_MINIO_CONSOLE_PORT={{ deepflow_minio_console_port }}
DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }}
DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }}
DEEPFLOW_S3_SECRET_KEY={{ deepflow_s3_secret_key }}
DEEPFLOW_S3_REGION={{ deepflow_s3_region }}

17
roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.svc Normal file
View File

@ -0,0 +1,17 @@
[Unit]
Description=DeepFlow ClickHouse + S3 (Docker Compose)
After=docker.service network-online.target
Wants=docker.service network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory={{ deepflow_clickhouse_s3_stack_dir }}
EnvironmentFile={{ deepflow_clickhouse_s3_env_file }}
ExecStart=/usr/bin/docker compose --env-file {{ deepflow_clickhouse_s3_env_file }} -f {{ deepflow_clickhouse_s3_compose_file }} up -d
ExecStop=/usr/bin/docker compose --env-file {{ deepflow_clickhouse_s3_env_file }} -f {{ deepflow_clickhouse_s3_compose_file }} down
TimeoutStartSec=0
TimeoutStopSec=120
[Install]
WantedBy=multi-user.target

33
roles/deepflow_clickhouse_s3/templates/docker-compose.yml.j2 Normal file
View File

@ -0,0 +1,33 @@
version: '3.9'
services:
clickhouse:
container_name: deepflow-clickhouse
image: ${DEEPFLOW_CLICKHOUSE_IMAGE}
restart: unless-stopped
environment:
CLICKHOUSE_DB: deepflow
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: '1'
ports:
- '${DEEPFLOW_CLICKHOUSE_HTTP_PORT}:8123'
- '${DEEPFLOW_CLICKHOUSE_TCP_PORT}:9000'
ulimits:
nofile:
soft: 262144
hard: 262144
volumes:
- ${DEEPFLOW_CLICKHOUSE_DATA}:/var/lib/clickhouse
minio:
container_name: deepflow-minio
image: ${DEEPFLOW_MINIO_IMAGE}
restart: unless-stopped
environment:
MINIO_ROOT_USER: ${DEEPFLOW_S3_ACCESS_KEY}
MINIO_ROOT_PASSWORD: ${DEEPFLOW_S3_SECRET_KEY}
command: server /data --console-address ':9001'
ports:
- '${DEEPFLOW_MINIO_API_PORT}:9000'
- '${DEEPFLOW_MINIO_CONSOLE_PORT}:9001'
volumes:
- ${DEEPFLOW_S3_DATA}:/data

10
roles/deepflow_mysql/README.md Normal file
View File

@ -0,0 +1,10 @@
# Role: deepflow_mysql
Deploy MySQL backend for DeepFlow with Docker Compose managed by systemd.
## Key Variables
- `deepflow_mysql_port` (default `13306`)
- `deepflow_mysql_database` (default `deepflow`)
- `deepflow_mysql_user` / `deepflow_mysql_password`
- `deepflow_mysql_root_password`

18
roles/deepflow_mysql/defaults/main.yml Normal file
View File

@ -0,0 +1,18 @@
---
#-----------------------------------------------------------------
# DEEPFLOW MYSQL
#-----------------------------------------------------------------
deepflow_mysql_enabled: true
deepflow_mysql_stack_dir: /opt/deepflow-mysql
deepflow_mysql_env_file: /etc/default/deepflow-mysql
deepflow_mysql_compose_file: "{{ deepflow_mysql_stack_dir }}/docker-compose.yml"
deepflow_mysql_data: /data/deepflow/mysql
deepflow_mysql_image: mysql:8.4
deepflow_mysql_port: 13306
deepflow_mysql_root_password: DeepFlow.Root.ChangeMe
deepflow_mysql_user: deepflow
deepflow_mysql_password: DeepFlow.MySQL.ChangeMe
deepflow_mysql_database: deepflow

7
roles/deepflow_mysql/meta/main.yml Normal file
View File

@ -0,0 +1,7 @@
galaxy_info:
author: observability.svc.plus
description: Deploy DeepFlow MySQL backend via Docker Compose
license: Apache-2.0
min_ansible_version: '2.10'
dependencies: []

77
roles/deepflow_mysql/tasks/main.yml Normal file
View File

@ -0,0 +1,77 @@
---
#--------------------------------------------------------------#
# Preflight [deepflow_mysql_check]
#--------------------------------------------------------------#
- name: check deepflow mysql prerequisites
tags: deepflow_mysql_check
block:
- name: assert docker is enabled
assert:
that:
- docker_enabled is defined
- docker_enabled | bool
fail_msg: "docker_enabled=true is required for deepflow_mysql role"
- name: check docker binary exists
command: docker --version
changed_when: false
#--------------------------------------------------------------#
# Configure [deepflow_mysql_conf]
#--------------------------------------------------------------#
- name: configure deepflow mysql stack
tags: deepflow_mysql_conf
block:
- name: create deepflow mysql directories
file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: '0755'
loop:
- "{{ deepflow_mysql_stack_dir }}"
- "{{ deepflow_mysql_data }}"
- name: render deepflow mysql environment
template:
src: deepflow-mysql.env.j2
dest: "{{ deepflow_mysql_env_file }}"
owner: root
group: root
mode: '0640'
- name: render deepflow mysql docker compose
template:
src: docker-compose.yml.j2
dest: "{{ deepflow_mysql_compose_file }}"
owner: root
group: root
mode: '0644'
- name: install deepflow mysql systemd unit
template:
src: deepflow-mysql.svc
dest: "{{ systemd_dir }}/deepflow-mysql.service"
owner: root
group: root
mode: '0644'
#--------------------------------------------------------------#
# Launch [deepflow_mysql_up]
#--------------------------------------------------------------#
- name: launch deepflow mysql stack
tags: deepflow_mysql_up
block:
- name: start deepflow mysql service
systemd:
name: deepflow-mysql
state: restarted
enabled: yes
daemon_reload: yes
- name: wait for deepflow mysql port
wait_for:
host: 127.0.0.1
port: "{{ deepflow_mysql_port }}"
timeout: 60

8
roles/deepflow_mysql/templates/deepflow-mysql.env.j2 Normal file
View File

@ -0,0 +1,8 @@
DEEPFLOW_MYSQL_IMAGE={{ deepflow_mysql_image }}
DEEPFLOW_MYSQL_PORT={{ deepflow_mysql_port }}
DEEPFLOW_MYSQL_DATA={{ deepflow_mysql_data }}
DEEPFLOW_MYSQL_ROOT_PASSWORD={{ deepflow_mysql_root_password }}
DEEPFLOW_MYSQL_USER={{ deepflow_mysql_user }}
DEEPFLOW_MYSQL_PASSWORD={{ deepflow_mysql_password }}
DEEPFLOW_MYSQL_DATABASE={{ deepflow_mysql_database }}

17
roles/deepflow_mysql/templates/deepflow-mysql.svc Normal file
View File

@ -0,0 +1,17 @@
[Unit]
Description=DeepFlow MySQL (Docker Compose)
After=docker.service network-online.target
Wants=docker.service network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory={{ deepflow_mysql_stack_dir }}
EnvironmentFile={{ deepflow_mysql_env_file }}
ExecStart=/usr/bin/docker compose --env-file {{ deepflow_mysql_env_file }} -f {{ deepflow_mysql_compose_file }} up -d
ExecStop=/usr/bin/docker compose --env-file {{ deepflow_mysql_env_file }} -f {{ deepflow_mysql_compose_file }} down
TimeoutStartSec=0
TimeoutStopSec=120
[Install]
WantedBy=multi-user.target

16
roles/deepflow_mysql/templates/docker-compose.yml.j2 Normal file
View File

@ -0,0 +1,16 @@
version: '3.9'
services:
mysql:
container_name: deepflow-mysql
image: ${DEEPFLOW_MYSQL_IMAGE}
restart: unless-stopped
environment:
MYSQL_ROOT_PASSWORD: ${DEEPFLOW_MYSQL_ROOT_PASSWORD}
MYSQL_DATABASE: ${DEEPFLOW_MYSQL_DATABASE}
MYSQL_USER: ${DEEPFLOW_MYSQL_USER}
MYSQL_PASSWORD: ${DEEPFLOW_MYSQL_PASSWORD}
ports:
- '${DEEPFLOW_MYSQL_PORT}:3306'
volumes:
- ${DEEPFLOW_MYSQL_DATA}:/var/lib/mysql

21
roles/deepflow_server/README.md Normal file
View File

@ -0,0 +1,21 @@
# Role: deepflow_server
Deploy DeepFlow control plane (`deepflow-server` + `deepflow-app`) with Docker Compose managed by systemd.
This role expects backend dependencies from separate roles:
- `deepflow_mysql`
- `deepflow_clickhouse_s3`
## Usage
1. Ensure Docker is installed (`./docker.yml`) and `docker_enabled: true`.
2. Run backend roles first, then this role (see `deepflow.yml`).
## Key Variables
- `deepflow_server_grpc_port` (default `20035`)
- `deepflow_server_http_port` (default `20417`)
- `deepflow_app_port` (default `20880`)
- `deepflow_clickhouse_addr` (default `host.docker.internal:19000`)
- `deepflow_s3_endpoint` (default `http://host.docker.internal:19090`)

32
roles/deepflow_server/defaults/main.yml Normal file
View File

@ -0,0 +1,32 @@
---
#-----------------------------------------------------------------
# DEEPFLOW SERVER
#-----------------------------------------------------------------
deepflow_enabled: true
deepflow_stack_dir: /opt/deepflow-server
deepflow_data: /data/deepflow
deepflow_env_file: /etc/default/deepflow-server
deepflow_compose_file: "{{ deepflow_stack_dir }}/docker-compose.yml"
# images (pin to specific tags before production)
deepflow_server_image: deepflowio/deepflow-server-ce:latest
deepflow_app_image: deepflowio/deepflow-app-ce:latest
# exposed ports
deepflow_server_grpc_port: 20035
deepflow_server_http_port: 20417
deepflow_app_port: 20880
# backend endpoints (provided by dedicated roles)
deepflow_clickhouse_addr: host.docker.internal:19000
deepflow_s3_endpoint: http://host.docker.internal:19090
deepflow_s3_bucket: deepflow
deepflow_s3_access_key: deepflow
deepflow_s3_secret_key: DeepFlow.S3.ChangeMe
deepflow_s3_region: us-east-1
deepflow_mysql_addr: host.docker.internal:13306
deepflow_mysql_user: deepflow
deepflow_mysql_password: DeepFlow.MySQL.ChangeMe
deepflow_mysql_database: deepflow

7
roles/deepflow_server/meta/main.yml Normal file
View File

@ -0,0 +1,7 @@
galaxy_info:
author: observability.svc.plus
description: Deploy DeepFlow stack via Docker Compose
license: Apache-2.0
min_ansible_version: '2.10'
dependencies: []

84
roles/deepflow_server/tasks/main.yml Normal file
View File

@ -0,0 +1,84 @@
---
#--------------------------------------------------------------#
# Preflight [deepflow_check]
#--------------------------------------------------------------#
- name: check deepflow prerequisites
tags: deepflow_check
block:
- name: assert docker is enabled
assert:
that:
- docker_enabled is defined
- docker_enabled | bool
fail_msg: "docker_enabled=true is required for deepflow_server role"
- name: check docker binary exists
command: docker --version
changed_when: false
#--------------------------------------------------------------#
# Configure [deepflow_conf]
#--------------------------------------------------------------#
- name: configure deepflow-server stack
tags: deepflow_conf
block:
- name: create deepflow directories
file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: '0755'
loop:
- "{{ deepflow_stack_dir }}"
- "{{ deepflow_data }}"
- "{{ deepflow_data }}/server"
- name: render deepflow environment
template:
src: deepflow.env.j2
dest: "{{ deepflow_env_file }}"
owner: root
group: root
mode: '0640'
- name: render deepflow docker compose
template:
src: docker-compose.yml.j2
dest: "{{ deepflow_compose_file }}"
owner: root
group: root
mode: '0644'
- name: install deepflow systemd unit
template:
src: deepflow-server.svc
dest: "{{ systemd_dir }}/deepflow-server.service"
owner: root
group: root
mode: '0644'
#--------------------------------------------------------------#
# Launch [deepflow_up]
#--------------------------------------------------------------#
- name: launch deepflow-server stack
tags: deepflow_up
block:
- name: start deepflow-server service
systemd:
name: deepflow-server
state: restarted
enabled: yes
daemon_reload: yes
- name: wait for deepflow gRPC port
wait_for:
host: 127.0.0.1
port: "{{ deepflow_server_grpc_port }}"
timeout: 60
- name: wait for deepflow app port
wait_for:
host: 127.0.0.1
port: "{{ deepflow_app_port }}"
timeout: 60

17
roles/deepflow_server/templates/deepflow-server.svc Normal file
View File

@ -0,0 +1,17 @@
[Unit]
Description=DeepFlow Server Stack (Docker Compose)
After=docker.service network-online.target
Wants=docker.service network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory={{ deepflow_stack_dir }}
EnvironmentFile={{ deepflow_env_file }}
ExecStart=/usr/bin/docker compose --env-file {{ deepflow_env_file }} -f {{ deepflow_compose_file }} up -d
ExecStop=/usr/bin/docker compose --env-file {{ deepflow_env_file }} -f {{ deepflow_compose_file }} down
TimeoutStartSec=0
TimeoutStopSec=120
[Install]
WantedBy=multi-user.target

21
roles/deepflow_server/templates/deepflow.env.j2 Normal file
View File

@ -0,0 +1,21 @@
DEEPFLOW_STACK_DIR={{ deepflow_stack_dir }}
DEEPFLOW_DATA={{ deepflow_data }}
DEEPFLOW_SERVER_IMAGE={{ deepflow_server_image }}
DEEPFLOW_APP_IMAGE={{ deepflow_app_image }}
DEEPFLOW_SERVER_GRPC_PORT={{ deepflow_server_grpc_port }}
DEEPFLOW_SERVER_HTTP_PORT={{ deepflow_server_http_port }}
DEEPFLOW_APP_PORT={{ deepflow_app_port }}
DEEPFLOW_CLICKHOUSE_ADDR={{ deepflow_clickhouse_addr }}
DEEPFLOW_S3_ENDPOINT={{ deepflow_s3_endpoint }}
DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }}
DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }}
DEEPFLOW_S3_SECRET_KEY={{ deepflow_s3_secret_key }}
DEEPFLOW_S3_REGION={{ deepflow_s3_region }}
DEEPFLOW_MYSQL_ADDR={{ deepflow_mysql_addr }}
DEEPFLOW_MYSQL_USER={{ deepflow_mysql_user }}
DEEPFLOW_MYSQL_PASSWORD={{ deepflow_mysql_password }}
DEEPFLOW_MYSQL_DATABASE={{ deepflow_mysql_database }}

36
roles/deepflow_server/templates/docker-compose.yml.j2 Normal file
View File

@ -0,0 +1,36 @@
version: '3.9'
services:
deepflow-server:
container_name: deepflow-server
image: ${DEEPFLOW_SERVER_IMAGE}
restart: unless-stopped
extra_hosts:
- 'host.docker.internal:host-gateway'
environment:
DEEPFLOW_CLICKHOUSE_ADDR: ${DEEPFLOW_CLICKHOUSE_ADDR}
DEEPFLOW_S3_ENDPOINT: ${DEEPFLOW_S3_ENDPOINT}
DEEPFLOW_S3_BUCKET: ${DEEPFLOW_S3_BUCKET}
DEEPFLOW_S3_ACCESS_KEY: ${DEEPFLOW_S3_ACCESS_KEY}
DEEPFLOW_S3_SECRET_KEY: ${DEEPFLOW_S3_SECRET_KEY}
DEEPFLOW_S3_REGION: ${DEEPFLOW_S3_REGION}
DEEPFLOW_MYSQL_ADDR: ${DEEPFLOW_MYSQL_ADDR}
DEEPFLOW_MYSQL_USER: ${DEEPFLOW_MYSQL_USER}
DEEPFLOW_MYSQL_PASSWORD: ${DEEPFLOW_MYSQL_PASSWORD}
DEEPFLOW_MYSQL_DATABASE: ${DEEPFLOW_MYSQL_DATABASE}
ports:
- '${DEEPFLOW_SERVER_GRPC_PORT}:20035'
- '${DEEPFLOW_SERVER_HTTP_PORT}:20417'
volumes:
- ${DEEPFLOW_DATA}/server:/var/lib/deepflow
deepflow-app:
container_name: deepflow-app
image: ${DEEPFLOW_APP_IMAGE}
restart: unless-stopped
depends_on:
- deepflow-server
environment:
DEEPFLOW_SERVER_ADDR: http://deepflow-server:20417
ports:
- '${DEEPFLOW_APP_PORT}:80'

4
roles/infra/defaults/main.yml
View File

@ -38,6 +38,10 @@ infra_extra_services: [] # extra services to be added on infra home pag
# CADDY vs NGINX
#-----------------------------------------------------------------
caddy_enabled: true # use caddy as ingress controller?
# Optional DeepFlow agent ingress: TLS gRPC -> Caddy -> deepflow-server(h2c)
deepflow_grpc_enabled: false # expose dedicated gRPC ingress for deepflow-agent?
deepflow_grpc_domain: deepflow-agent.svc.plus
deepflow_grpc_upstream: 127.0.0.1:20035
nginx_enabled: false # enable nginx on this infra node?
nginx_clean: false # clean existing nginx config during init?
nginx_exporter_enabled: true # enable nginx_exporter on this infra node?

11
roles/infra/templates/caddy/Caddyfile
View File

@ -150,3 +150,14 @@
file_server
}
}
{% if deepflow_grpc_enabled | default(false) %}
{{ deepflow_grpc_domain | default('deepflow-agent.svc.plus') }} {
# TLS terminates at Caddy, then proxy h2c gRPC to deepflow-server.
reverse_proxy {{ deepflow_grpc_upstream | default('127.0.0.1:20035') }} {
transport http {
versions h2c
}
}
}
{% endif %}

122
scripts/agent-install.sh
View File

@ -17,6 +17,11 @@ METRICS_ENDPOINT=""
LOGS_ENDPOINT=""
METRICS_ENDPOINT_SET=false
LOGS_ENDPOINT_SET=false
DEEPFLOW_AGENT_ENABLED=false
DEEPFLOW_GRPC_ENDPOINT=""
DEEPFLOW_AGENT_ENDPOINT_ARG="--grpc-server"
DEEPFLOW_AGENT_DOWNLOAD_URL=""
DEEPFLOW_AGENT_BIN="${BIN_DIR}/deepflow-agent"
AUTO_YES=false
GREEN='\033[0;32m'
@ -46,6 +51,11 @@ Options:
--endpoint URL Base ingest endpoint (default: ${DEFAULT_ENDPOINT})
--metrics-endpoint URL Prometheus remote_write endpoint (optional override)
--logs-endpoint URL Loki push endpoint (optional override)
--deepflow-agent Install and enable deepflow-agent service
--deepflow-grpc-endpoint HOST:PORT DeepFlow gRPC endpoint (required if --deepflow-agent)
--deepflow-agent-endpoint-arg ARG deepflow-agent endpoint arg (default: ${DEEPFLOW_AGENT_ENDPOINT_ARG})
--deepflow-agent-download-url URL Download deepflow-agent binary to ${DEEPFLOW_AGENT_BIN}
--deepflow-agent-bin PATH Use existing deepflow-agent binary path
-y, --yes Non-interactive mode
-h, --help Show help
@ -101,6 +111,42 @@ while [[ $# -gt 0 ]]; do
LOGS_ENDPOINT_SET=true
shift
;;
--deepflow-agent)
DEEPFLOW_AGENT_ENABLED=true
shift
;;
--deepflow-grpc-endpoint)
DEEPFLOW_GRPC_ENDPOINT="$2"
shift 2
;;
--deepflow-grpc-endpoint=*)
DEEPFLOW_GRPC_ENDPOINT="${1#*=}"
shift
;;
--deepflow-agent-endpoint-arg)
DEEPFLOW_AGENT_ENDPOINT_ARG="$2"
shift 2
;;
--deepflow-agent-endpoint-arg=*)
DEEPFLOW_AGENT_ENDPOINT_ARG="${1#*=}"
shift
;;
--deepflow-agent-download-url)
DEEPFLOW_AGENT_DOWNLOAD_URL="$2"
shift 2
;;
--deepflow-agent-download-url=*)
DEEPFLOW_AGENT_DOWNLOAD_URL="${1#*=}"
shift
;;
--deepflow-agent-bin)
DEEPFLOW_AGENT_BIN="$2"
shift 2
;;
--deepflow-agent-bin=*)
DEEPFLOW_AGENT_BIN="${1#*=}"
shift
;;
-y|--yes)
AUTO_YES=true
shift
@ -127,6 +173,9 @@ fi
if [[ -z "${LOGS_ENDPOINT}" ]]; then
LOGS_ENDPOINT="${base_endpoint}/ingest/logs/insert"
fi
if [[ "${DEEPFLOW_AGENT_ENABLED}" == "true" && -z "${DEEPFLOW_GRPC_ENDPOINT}" ]]; then
DEEPFLOW_GRPC_ENDPOINT="deepflow-agent.${base_endpoint#*://}:443"
fi
# observability server should bypass external HTTPS ingress for local self-monitoring
local_host="$(hostname -f 2>/dev/null || hostname)"
@ -382,17 +431,76 @@ WantedBy=multi-user.target"
systemctl restart vector
}
install_deepflow_agent() {
if [[ "${DEEPFLOW_AGENT_ENABLED}" != "true" ]]; then
return 0
fi
if [[ -z "${DEEPFLOW_GRPC_ENDPOINT}" ]]; then
log_error "DeepFlow agent enabled but --deepflow-grpc-endpoint is empty."
exit 1
fi
if [[ -n "${DEEPFLOW_AGENT_DOWNLOAD_URL}" ]]; then
log_info "Downloading deepflow-agent binary..."
curl -fL --progress-bar "${DEEPFLOW_AGENT_DOWNLOAD_URL}" -o "${DEEPFLOW_AGENT_BIN}"
chmod 0755 "${DEEPFLOW_AGENT_BIN}"
elif [[ ! -x "${DEEPFLOW_AGENT_BIN}" ]]; then
if command -v deepflow-agent >/dev/null 2>&1; then
DEEPFLOW_AGENT_BIN="$(command -v deepflow-agent)"
else
log_error "deepflow-agent binary not found. Use --deepflow-agent-download-url or --deepflow-agent-bin."
exit 1
fi
fi
cat <<EOF > "${CONFIG_DIR}/deepflow-agent.env"
DEEPFLOW_GRPC_ENDPOINT=${DEEPFLOW_GRPC_ENDPOINT}
DEEPFLOW_AGENT_ENDPOINT_ARG=${DEEPFLOW_AGENT_ENDPOINT_ARG}
EOF
cat <<'EOF' > "${BIN_DIR}/run-deepflow-agent.sh"
#!/bin/bash
set -euo pipefail
: "${DEEPFLOW_AGENT_BIN:?missing DEEPFLOW_AGENT_BIN}"
: "${DEEPFLOW_AGENT_ENDPOINT_ARG:?missing DEEPFLOW_AGENT_ENDPOINT_ARG}"
: "${DEEPFLOW_GRPC_ENDPOINT:?missing DEEPFLOW_GRPC_ENDPOINT}"
exec "${DEEPFLOW_AGENT_BIN}" "${DEEPFLOW_AGENT_ENDPOINT_ARG}" "${DEEPFLOW_GRPC_ENDPOINT}"
EOF
chmod 0755 "${BIN_DIR}/run-deepflow-agent.sh"
write_unit_if_changed "deepflow_agent" "[Unit]
Description=DeepFlow Agent
After=network-online.target
Wants=network-online.target
[Service]
User=root
EnvironmentFile=${CONFIG_DIR}/deepflow-agent.env
Environment=DEEPFLOW_AGENT_BIN=${DEEPFLOW_AGENT_BIN}
ExecStart=${BIN_DIR}/run-deepflow-agent.sh
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target"
systemctl enable --now deepflow_agent
systemctl restart deepflow_agent
}
uninstall_agent() {
confirm "This will uninstall observability agent components. Continue?" || {
log_info "Cancelled."
return 0
}
for svc in vector process_exporter node_exporter; do
for svc in deepflow_agent vector process_exporter node_exporter; do
systemctl disable --now "${svc}" >/dev/null 2>&1 || true
rm -f "/etc/systemd/system/${svc}.service"
done
systemctl daemon-reload
rm -f "${BIN_DIR}/run-deepflow-agent.sh" "${CONFIG_DIR}/deepflow-agent.env"
rm -rf "${INSTALL_DIR}"
log_success "Agent components uninstalled."
}
@ -408,6 +516,14 @@ verify_installation() {
systemctl status "${service}" --no-pager | head -n 20 || true
fi
done
if [[ "${DEEPFLOW_AGENT_ENABLED}" == "true" ]]; then
if systemctl is-active --quiet deepflow_agent; then
log_success "Service 'deepflow_agent' is running"
else
log_fail "Service 'deepflow_agent' is NOT running"
systemctl status deepflow_agent --no-pager | head -n 20 || true
fi
fi
log_info "Checking ports..."
for item in "9100 Node Exporter" "9256 Process Exporter"; do
@ -435,9 +551,13 @@ deploy_agent() {
log_info "Base endpoint=${ENDPOINT}"
log_info "Metrics endpoint=${METRICS_ENDPOINT}"
log_info "Logs endpoint=${LOGS_ENDPOINT}"
if [[ "${DEEPFLOW_AGENT_ENABLED}" == "true" ]]; then
log_info "DeepFlow endpoint=${DEEPFLOW_GRPC_ENDPOINT}"
fi
install_node_exporter
install_process_exporter
install_vector
install_deepflow_agent
verify_installation
log_success "Agent deploy/upgrade complete."
print_endpoint_summary