refactor: split deepflow into server mysql and clickhouse-s3 roles

This commit is contained in:
Haitao Pan 2026-02-04 10:08:54 +08:00
parent e49c78aa90
commit 89eee66a15
21 changed files with 405 additions and 73 deletions

View File

@ -15,7 +15,7 @@
# vi pigsty.yml # IMPORTANT: CHANGE CREDENTIALS / DOMAIN
# ./deploy.yml # install infra stack
# ./docker.yml # install docker & docker-compose
# ./deepflow.yml # install deepflow with deepflow_server role
# ./deepflow.yml # install deepflow with three roles
all:
children:
@ -24,22 +24,36 @@ all:
hosts: { 10.10.10.10: {} }
vars:
deepflow_enabled: true
deepflow_stack_dir: /opt/deepflow-server
deepflow_mysql_enabled: true
deepflow_clickhouse_s3_enabled: true
deepflow_data: /data/deepflow
deepflow_server_grpc_port: 20035
deepflow_server_http_port: 20417
deepflow_app_port: 20880
# role: deepflow_mysql
deepflow_mysql_port: 13306
deepflow_mysql_root_password: DeepFlow.Root.ChangeMe
deepflow_mysql_user: deepflow
deepflow_mysql_password: DeepFlow.MySQL.ChangeMe
deepflow_mysql_database: deepflow
# role: deepflow_clickhouse_s3
deepflow_clickhouse_http_port: 18123
deepflow_clickhouse_tcp_port: 19000
deepflow_minio_api_port: 19090
deepflow_minio_console_port: 19091
deepflow_s3_bucket: deepflow
deepflow_s3_access_key: deepflow
deepflow_s3_secret_key: DeepFlow.S3.ChangeMe
deepflow_s3_region: us-east-1
# role: deepflow_server
deepflow_server_grpc_port: 20035
deepflow_server_http_port: 20417
deepflow_app_port: 20880
deepflow_clickhouse_addr: host.docker.internal:19000
deepflow_s3_endpoint: http://host.docker.internal:19090
deepflow_mysql_addr: host.docker.internal:13306
infra: { hosts: { 10.10.10.10: { infra_seq: 1 } } }
etcd: { hosts: { 10.10.10.10: { etcd_seq: 1 } }, vars: { etcd_cluster: etcd } }

View File

@ -2,21 +2,23 @@
---
#==============================================================#
# File : deepflow.yml
# Desc : deploy deepflow server stack
# Desc : deploy deepflow stack with three dedicated roles
# Ctime : 2026-02-04
# Mtime : 2026-02-04
# Path : deepflow.yml
# License : Apache-2.0 @ https://pigsty.io/docs/about/license/
#==============================================================#
- name: DEEPFLOW SERVER
- name: DEEPFLOW STACK
become: true
hosts: all
gather_facts: no
roles:
- { role: node_id , tags: node-id, when: deepflow_enabled | default(true) | bool }
- { role: deepflow_server , tags: deepflow_server, when: deepflow_enabled | default(true) | bool }
- { role: node_id , tags: node-id, when: deepflow_enabled | default(true) | bool }
- { role: deepflow_mysql , tags: deepflow_mysql, when: deepflow_mysql_enabled | default(true) | bool }
- { role: deepflow_clickhouse_s3, tags: deepflow_clickhouse_s3, when: deepflow_clickhouse_s3_enabled | default(true) | bool }
- { role: deepflow_server , tags: deepflow_server, when: deepflow_enabled | default(true) | bool }
# Usage:
# 1. Define deepflow group in pigsty.yml

View File

@ -0,0 +1,10 @@
# Role: deepflow_clickhouse_s3
Deploy ClickHouse + MinIO(S3) backend for DeepFlow with Docker Compose managed by systemd.
## Key Variables
- `deepflow_clickhouse_tcp_port` (default `19000`)
- `deepflow_clickhouse_http_port` (default `18123`)
- `deepflow_minio_api_port` (default `19090`)
- `deepflow_s3_access_key` / `deepflow_s3_secret_key`

View File

@ -0,0 +1,26 @@
---
#-----------------------------------------------------------------
# DEEPFLOW CLICKHOUSE + S3
#-----------------------------------------------------------------
deepflow_clickhouse_s3_enabled: true
deepflow_clickhouse_s3_stack_dir: /opt/deepflow-clickhouse-s3
deepflow_clickhouse_s3_env_file: /etc/default/deepflow-clickhouse-s3
deepflow_clickhouse_s3_compose_file: "{{ deepflow_clickhouse_s3_stack_dir }}/docker-compose.yml"
deepflow_data: /data/deepflow
deepflow_clickhouse_data: "{{ deepflow_data }}/clickhouse"
deepflow_s3_data: "{{ deepflow_data }}/minio"
deepflow_clickhouse_image: clickhouse/clickhouse-server:24.8
deepflow_minio_image: minio/minio:RELEASE.2025-01-20T14-49-07Z
deepflow_clickhouse_http_port: 18123
deepflow_clickhouse_tcp_port: 19000
deepflow_minio_api_port: 19090
deepflow_minio_console_port: 19091
deepflow_s3_bucket: deepflow
deepflow_s3_access_key: deepflow
deepflow_s3_secret_key: DeepFlow.S3.ChangeMe
deepflow_s3_region: us-east-1

View File

@ -0,0 +1,7 @@
galaxy_info:
author: observability.svc.plus
description: Deploy DeepFlow ClickHouse + S3 backend via Docker Compose
license: Apache-2.0
min_ansible_version: '2.10'
dependencies: []

View File

@ -0,0 +1,85 @@
---
#----------------------------------------------------------------#
# Preflight [deepflow_clickhouse_check]
#----------------------------------------------------------------#
- name: check deepflow clickhouse+s3 prerequisites
tags: deepflow_clickhouse_check
block:
- name: assert docker is enabled
assert:
that:
- docker_enabled is defined
- docker_enabled | bool
fail_msg: "docker_enabled=true is required for deepflow_clickhouse_s3 role"
- name: check docker binary exists
command: docker --version
changed_when: false
#----------------------------------------------------------------#
# Configure [deepflow_clickhouse_conf]
#----------------------------------------------------------------#
- name: configure deepflow clickhouse+s3 stack
tags: deepflow_clickhouse_conf
block:
- name: create deepflow clickhouse+s3 directories
file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: '0755'
loop:
- "{{ deepflow_clickhouse_s3_stack_dir }}"
- "{{ deepflow_data }}"
- "{{ deepflow_clickhouse_data }}"
- "{{ deepflow_s3_data }}"
- name: render deepflow clickhouse+s3 environment
template:
src: deepflow-clickhouse-s3.env.j2
dest: "{{ deepflow_clickhouse_s3_env_file }}"
owner: root
group: root
mode: '0640'
- name: render deepflow clickhouse+s3 docker compose
template:
src: docker-compose.yml.j2
dest: "{{ deepflow_clickhouse_s3_compose_file }}"
owner: root
group: root
mode: '0644'
- name: install deepflow clickhouse+s3 systemd unit
template:
src: deepflow-clickhouse-s3.svc
dest: "{{ systemd_dir }}/deepflow-clickhouse-s3.service"
owner: root
group: root
mode: '0644'
#----------------------------------------------------------------#
# Launch [deepflow_clickhouse_up]
#----------------------------------------------------------------#
- name: launch deepflow clickhouse+s3 stack
tags: deepflow_clickhouse_up
block:
- name: start deepflow clickhouse+s3 service
systemd:
name: deepflow-clickhouse-s3
state: restarted
enabled: yes
daemon_reload: yes
- name: wait for deepflow clickhouse tcp port
wait_for:
host: 127.0.0.1
port: "{{ deepflow_clickhouse_tcp_port }}"
timeout: 60
- name: wait for deepflow minio api port
wait_for:
host: 127.0.0.1
port: "{{ deepflow_minio_api_port }}"
timeout: 60

View File

@ -0,0 +1,15 @@
DEEPFLOW_CLICKHOUSE_IMAGE={{ deepflow_clickhouse_image }}
DEEPFLOW_MINIO_IMAGE={{ deepflow_minio_image }}
DEEPFLOW_CLICKHOUSE_DATA={{ deepflow_clickhouse_data }}
DEEPFLOW_S3_DATA={{ deepflow_s3_data }}
DEEPFLOW_CLICKHOUSE_HTTP_PORT={{ deepflow_clickhouse_http_port }}
DEEPFLOW_CLICKHOUSE_TCP_PORT={{ deepflow_clickhouse_tcp_port }}
DEEPFLOW_MINIO_API_PORT={{ deepflow_minio_api_port }}
DEEPFLOW_MINIO_CONSOLE_PORT={{ deepflow_minio_console_port }}
DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }}
DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }}
DEEPFLOW_S3_SECRET_KEY={{ deepflow_s3_secret_key }}
DEEPFLOW_S3_REGION={{ deepflow_s3_region }}

View File

@ -0,0 +1,17 @@
[Unit]
Description=DeepFlow ClickHouse + S3 (Docker Compose)
After=docker.service network-online.target
Wants=docker.service network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory={{ deepflow_clickhouse_s3_stack_dir }}
EnvironmentFile={{ deepflow_clickhouse_s3_env_file }}
ExecStart=/usr/bin/docker compose --env-file {{ deepflow_clickhouse_s3_env_file }} -f {{ deepflow_clickhouse_s3_compose_file }} up -d
ExecStop=/usr/bin/docker compose --env-file {{ deepflow_clickhouse_s3_env_file }} -f {{ deepflow_clickhouse_s3_compose_file }} down
TimeoutStartSec=0
TimeoutStopSec=120
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,33 @@
version: '3.9'
services:
clickhouse:
container_name: deepflow-clickhouse
image: ${DEEPFLOW_CLICKHOUSE_IMAGE}
restart: unless-stopped
environment:
CLICKHOUSE_DB: deepflow
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: '1'
ports:
- '${DEEPFLOW_CLICKHOUSE_HTTP_PORT}:8123'
- '${DEEPFLOW_CLICKHOUSE_TCP_PORT}:9000'
ulimits:
nofile:
soft: 262144
hard: 262144
volumes:
- ${DEEPFLOW_CLICKHOUSE_DATA}:/var/lib/clickhouse
minio:
container_name: deepflow-minio
image: ${DEEPFLOW_MINIO_IMAGE}
restart: unless-stopped
environment:
MINIO_ROOT_USER: ${DEEPFLOW_S3_ACCESS_KEY}
MINIO_ROOT_PASSWORD: ${DEEPFLOW_S3_SECRET_KEY}
command: server /data --console-address ':9001'
ports:
- '${DEEPFLOW_MINIO_API_PORT}:9000'
- '${DEEPFLOW_MINIO_CONSOLE_PORT}:9001'
volumes:
- ${DEEPFLOW_S3_DATA}:/data

View File

@ -0,0 +1,10 @@
# Role: deepflow_mysql
Deploy MySQL backend for DeepFlow with Docker Compose managed by systemd.
## Key Variables
- `deepflow_mysql_port` (default `13306`)
- `deepflow_mysql_database` (default `deepflow`)
- `deepflow_mysql_user` / `deepflow_mysql_password`
- `deepflow_mysql_root_password`

View File

@ -0,0 +1,18 @@
---
#-----------------------------------------------------------------
# DEEPFLOW MYSQL
#-----------------------------------------------------------------
deepflow_mysql_enabled: true
deepflow_mysql_stack_dir: /opt/deepflow-mysql
deepflow_mysql_env_file: /etc/default/deepflow-mysql
deepflow_mysql_compose_file: "{{ deepflow_mysql_stack_dir }}/docker-compose.yml"
deepflow_mysql_data: /data/deepflow/mysql
deepflow_mysql_image: mysql:8.4
deepflow_mysql_port: 13306
deepflow_mysql_root_password: DeepFlow.Root.ChangeMe
deepflow_mysql_user: deepflow
deepflow_mysql_password: DeepFlow.MySQL.ChangeMe
deepflow_mysql_database: deepflow

View File

@ -0,0 +1,7 @@
galaxy_info:
author: observability.svc.plus
description: Deploy DeepFlow MySQL backend via Docker Compose
license: Apache-2.0
min_ansible_version: '2.10'
dependencies: []

View File

@ -0,0 +1,77 @@
---
#--------------------------------------------------------------#
# Preflight [deepflow_mysql_check]
#--------------------------------------------------------------#
- name: check deepflow mysql prerequisites
tags: deepflow_mysql_check
block:
- name: assert docker is enabled
assert:
that:
- docker_enabled is defined
- docker_enabled | bool
fail_msg: "docker_enabled=true is required for deepflow_mysql role"
- name: check docker binary exists
command: docker --version
changed_when: false
#--------------------------------------------------------------#
# Configure [deepflow_mysql_conf]
#--------------------------------------------------------------#
- name: configure deepflow mysql stack
tags: deepflow_mysql_conf
block:
- name: create deepflow mysql directories
file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: '0755'
loop:
- "{{ deepflow_mysql_stack_dir }}"
- "{{ deepflow_mysql_data }}"
- name: render deepflow mysql environment
template:
src: deepflow-mysql.env.j2
dest: "{{ deepflow_mysql_env_file }}"
owner: root
group: root
mode: '0640'
- name: render deepflow mysql docker compose
template:
src: docker-compose.yml.j2
dest: "{{ deepflow_mysql_compose_file }}"
owner: root
group: root
mode: '0644'
- name: install deepflow mysql systemd unit
template:
src: deepflow-mysql.svc
dest: "{{ systemd_dir }}/deepflow-mysql.service"
owner: root
group: root
mode: '0644'
#--------------------------------------------------------------#
# Launch [deepflow_mysql_up]
#--------------------------------------------------------------#
- name: launch deepflow mysql stack
tags: deepflow_mysql_up
block:
- name: start deepflow mysql service
systemd:
name: deepflow-mysql
state: restarted
enabled: yes
daemon_reload: yes
- name: wait for deepflow mysql port
wait_for:
host: 127.0.0.1
port: "{{ deepflow_mysql_port }}"
timeout: 60

View File

@ -0,0 +1,8 @@
DEEPFLOW_MYSQL_IMAGE={{ deepflow_mysql_image }}
DEEPFLOW_MYSQL_PORT={{ deepflow_mysql_port }}
DEEPFLOW_MYSQL_DATA={{ deepflow_mysql_data }}
DEEPFLOW_MYSQL_ROOT_PASSWORD={{ deepflow_mysql_root_password }}
DEEPFLOW_MYSQL_USER={{ deepflow_mysql_user }}
DEEPFLOW_MYSQL_PASSWORD={{ deepflow_mysql_password }}
DEEPFLOW_MYSQL_DATABASE={{ deepflow_mysql_database }}

View File

@ -0,0 +1,17 @@
[Unit]
Description=DeepFlow MySQL (Docker Compose)
After=docker.service network-online.target
Wants=docker.service network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory={{ deepflow_mysql_stack_dir }}
EnvironmentFile={{ deepflow_mysql_env_file }}
ExecStart=/usr/bin/docker compose --env-file {{ deepflow_mysql_env_file }} -f {{ deepflow_mysql_compose_file }} up -d
ExecStop=/usr/bin/docker compose --env-file {{ deepflow_mysql_env_file }} -f {{ deepflow_mysql_compose_file }} down
TimeoutStartSec=0
TimeoutStopSec=120
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,16 @@
version: '3.9'
services:
mysql:
container_name: deepflow-mysql
image: ${DEEPFLOW_MYSQL_IMAGE}
restart: unless-stopped
environment:
MYSQL_ROOT_PASSWORD: ${DEEPFLOW_MYSQL_ROOT_PASSWORD}
MYSQL_DATABASE: ${DEEPFLOW_MYSQL_DATABASE}
MYSQL_USER: ${DEEPFLOW_MYSQL_USER}
MYSQL_PASSWORD: ${DEEPFLOW_MYSQL_PASSWORD}
ports:
- '${DEEPFLOW_MYSQL_PORT}:3306'
volumes:
- ${DEEPFLOW_MYSQL_DATA}:/var/lib/mysql

View File

@ -1,23 +1,21 @@
# Role: deepflow_server
Deploy DeepFlow server stack (deepflow-server + deepflow-app + ClickHouse + MinIO)
with Docker Compose managed by systemd.
Deploy DeepFlow control plane (`deepflow-server` + `deepflow-app`) with Docker Compose managed by systemd.
This role expects backend dependencies from separate roles:
- `deepflow_mysql`
- `deepflow_clickhouse_s3`
## Usage
1. Ensure Docker is installed (`./docker.yml`) and `docker_enabled: true`.
2. Add hosts to a `deepflow` group with proper vars.
3. Run `./deepflow.yml -l deepflow`.
2. Run backend roles first, then this role (see `deepflow.yml`).
## Key Variables
- `deepflow_stack_dir` (default `/opt/deepflow-server`)
- `deepflow_data` (default `/data/deepflow`)
- `deepflow_server_grpc_port` (default `20035`)
- `deepflow_server_http_port` (default `20417`)
- `deepflow_app_port` (default `20880`)
- `deepflow_s3_access_key`, `deepflow_s3_secret_key`
## Related
- `roles/infra/templates/caddy/Caddyfile` for TLS gRPC ingress
- `roles/infra/defaults/main.yml` (`deepflow_grpc_*` variables)
- `deepflow_clickhouse_addr` (default `host.docker.internal:19000`)
- `deepflow_s3_endpoint` (default `http://host.docker.internal:19090`)

View File

@ -12,20 +12,21 @@ deepflow_compose_file: "{{ deepflow_stack_dir }}/docker-compose.yml"
# images (pin to specific tags before production)
deepflow_server_image: deepflowio/deepflow-server-ce:latest
deepflow_app_image: deepflowio/deepflow-app-ce:latest
deepflow_clickhouse_image: clickhouse/clickhouse-server:24.8
deepflow_minio_image: minio/minio:RELEASE.2025-01-20T14-49-07Z
# exposed ports
deepflow_server_grpc_port: 20035
deepflow_server_http_port: 20417
deepflow_app_port: 20880
deepflow_clickhouse_http_port: 18123
deepflow_clickhouse_tcp_port: 19000
deepflow_minio_api_port: 19090
deepflow_minio_console_port: 19091
# S3 backend used by deepflow-server
# backend endpoints (provided by dedicated roles)
deepflow_clickhouse_addr: host.docker.internal:19000
deepflow_s3_endpoint: http://host.docker.internal:19090
deepflow_s3_bucket: deepflow
deepflow_s3_access_key: deepflow
deepflow_s3_secret_key: DeepFlow.S3.ChangeMe
deepflow_s3_region: us-east-1
deepflow_mysql_addr: host.docker.internal:13306
deepflow_mysql_user: deepflow
deepflow_mysql_password: DeepFlow.MySQL.ChangeMe
deepflow_mysql_database: deepflow

View File

@ -32,8 +32,6 @@
loop:
- "{{ deepflow_stack_dir }}"
- "{{ deepflow_data }}"
- "{{ deepflow_data }}/clickhouse"
- "{{ deepflow_data }}/minio"
- "{{ deepflow_data }}/server"
- name: render deepflow environment

View File

@ -3,18 +3,19 @@ DEEPFLOW_DATA={{ deepflow_data }}
DEEPFLOW_SERVER_IMAGE={{ deepflow_server_image }}
DEEPFLOW_APP_IMAGE={{ deepflow_app_image }}
DEEPFLOW_CLICKHOUSE_IMAGE={{ deepflow_clickhouse_image }}
DEEPFLOW_MINIO_IMAGE={{ deepflow_minio_image }}
DEEPFLOW_SERVER_GRPC_PORT={{ deepflow_server_grpc_port }}
DEEPFLOW_SERVER_HTTP_PORT={{ deepflow_server_http_port }}
DEEPFLOW_APP_PORT={{ deepflow_app_port }}
DEEPFLOW_CLICKHOUSE_HTTP_PORT={{ deepflow_clickhouse_http_port }}
DEEPFLOW_CLICKHOUSE_TCP_PORT={{ deepflow_clickhouse_tcp_port }}
DEEPFLOW_MINIO_API_PORT={{ deepflow_minio_api_port }}
DEEPFLOW_MINIO_CONSOLE_PORT={{ deepflow_minio_console_port }}
DEEPFLOW_CLICKHOUSE_ADDR={{ deepflow_clickhouse_addr }}
DEEPFLOW_S3_ENDPOINT={{ deepflow_s3_endpoint }}
DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }}
DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }}
DEEPFLOW_S3_SECRET_KEY={{ deepflow_s3_secret_key }}
DEEPFLOW_S3_REGION={{ deepflow_s3_region }}
DEEPFLOW_MYSQL_ADDR={{ deepflow_mysql_addr }}
DEEPFLOW_MYSQL_USER={{ deepflow_mysql_user }}
DEEPFLOW_MYSQL_PASSWORD={{ deepflow_mysql_password }}
DEEPFLOW_MYSQL_DATABASE={{ deepflow_mysql_database }}

View File

@ -1,51 +1,23 @@
version: '3.9'
services:
clickhouse:
container_name: deepflow-clickhouse
image: ${DEEPFLOW_CLICKHOUSE_IMAGE}
restart: unless-stopped
environment:
CLICKHOUSE_DB: deepflow
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
ports:
- '${DEEPFLOW_CLICKHOUSE_HTTP_PORT}:8123'
- '${DEEPFLOW_CLICKHOUSE_TCP_PORT}:9000'
ulimits:
nofile:
soft: 262144
hard: 262144
volumes:
- ${DEEPFLOW_DATA}/clickhouse:/var/lib/clickhouse
minio:
container_name: deepflow-minio
image: ${DEEPFLOW_MINIO_IMAGE}
restart: unless-stopped
environment:
MINIO_ROOT_USER: ${DEEPFLOW_S3_ACCESS_KEY}
MINIO_ROOT_PASSWORD: ${DEEPFLOW_S3_SECRET_KEY}
command: server /data --console-address ':9001'
ports:
- '${DEEPFLOW_MINIO_API_PORT}:9000'
- '${DEEPFLOW_MINIO_CONSOLE_PORT}:9001'
volumes:
- ${DEEPFLOW_DATA}/minio:/data
deepflow-server:
container_name: deepflow-server
image: ${DEEPFLOW_SERVER_IMAGE}
restart: unless-stopped
depends_on:
- clickhouse
- minio
extra_hosts:
- 'host.docker.internal:host-gateway'
environment:
DEEPFLOW_CLICKHOUSE_ADDR: clickhouse:9000
DEEPFLOW_S3_ENDPOINT: http://minio:9000
DEEPFLOW_CLICKHOUSE_ADDR: ${DEEPFLOW_CLICKHOUSE_ADDR}
DEEPFLOW_S3_ENDPOINT: ${DEEPFLOW_S3_ENDPOINT}
DEEPFLOW_S3_BUCKET: ${DEEPFLOW_S3_BUCKET}
DEEPFLOW_S3_ACCESS_KEY: ${DEEPFLOW_S3_ACCESS_KEY}
DEEPFLOW_S3_SECRET_KEY: ${DEEPFLOW_S3_SECRET_KEY}
DEEPFLOW_S3_REGION: ${DEEPFLOW_S3_REGION}
DEEPFLOW_MYSQL_ADDR: ${DEEPFLOW_MYSQL_ADDR}
DEEPFLOW_MYSQL_USER: ${DEEPFLOW_MYSQL_USER}
DEEPFLOW_MYSQL_PASSWORD: ${DEEPFLOW_MYSQL_PASSWORD}
DEEPFLOW_MYSQL_DATABASE: ${DEEPFLOW_MYSQL_DATABASE}
ports:
- '${DEEPFLOW_SERVER_GRPC_PORT}:20035'
- '${DEEPFLOW_SERVER_HTTP_PORT}:20417'