feat: add deepflow_server ansible role and caddy grpc ingress

conf/app/deepflow.yml

@@ -0,0 +1,78 @@
+---
+#==============================================================#
+# File      :   deepflow.yml
+# Desc      :   observability config for running DeepFlow stack
+# Ctime     :   2026-02-04
+# Mtime     :   2026-02-04
+# License   :   Apache-2.0 @ https://pigsty.io/docs/about/license/
+#==============================================================#
+
+# how to use this template:
+#
+#  curl -fsSL https://repo.pigsty.io/get | bash; cd ~/pigsty
+# ./bootstrap                  # prepare local repo & ansible
+# ./configure -c app/deepflow  # use this deepflow config template
+# vi pigsty.yml                # IMPORTANT: CHANGE CREDENTIALS / DOMAIN
+# ./deploy.yml                 # install infra stack
+# ./docker.yml                 # install docker & docker-compose
+# ./deepflow.yml               # install deepflow with deepflow_server role
+
+all:
+  children:
+
+    deepflow:
+      hosts: { 10.10.10.10: {} }
+      vars:
+        deepflow_enabled: true
+        deepflow_stack_dir: /opt/deepflow-server
+        deepflow_data: /data/deepflow
+
+        deepflow_server_grpc_port: 20035
+        deepflow_server_http_port: 20417
+        deepflow_app_port: 20880
+        deepflow_clickhouse_http_port: 18123
+        deepflow_clickhouse_tcp_port: 19000
+        deepflow_minio_api_port: 19090
+        deepflow_minio_console_port: 19091
+
+        deepflow_s3_bucket: deepflow
+        deepflow_s3_access_key: deepflow
+        deepflow_s3_secret_key: DeepFlow.S3.ChangeMe
+        deepflow_s3_region: us-east-1
+
+    infra: { hosts: { 10.10.10.10: { infra_seq: 1 } } }
+    etcd:  { hosts: { 10.10.10.10: { etcd_seq: 1 } }, vars: { etcd_cluster: etcd } }
+
+  vars:
+    version: v4.0.0
+    admin_ip: 10.10.10.10
+    region: default
+    node_tune: oltp
+    pg_conf: oltp.yml
+    docker_enabled: true
+
+    # Caddy gRPC ingress for deepflow-agent:
+    caddy_enabled: true
+    deepflow_grpc_enabled: true
+    deepflow_grpc_domain: deepflow-agent.pigsty
+    deepflow_grpc_upstream: 127.0.0.1:20035
+
+    infra_portal:
+      home     : { domain: i.pigsty }
+      deepflow : { domain: deepflow.pigsty ,endpoint: "10.10.10.10:20880" }
+
+    proxy_env:
+      no_proxy: "localhost,127.0.0.1,10.0.0.0/8,192.168.0.0/16,*.pigsty,*.aliyun.com,mirrors.*,*.tsinghua.edu.cn"
+
+    repo_enabled: false
+    node_repo_modules: node,infra,pgsql
+
+    grafana_admin_password: pigsty
+    grafana_view_password: DBUser.Viewer
+    pg_admin_password: DBUser.DBA
+    pg_monitor_password: DBUser.Monitor
+    pg_replication_password: DBUser.Replicator
+    patroni_password: Patroni.API
+    haproxy_admin_password: pigsty
+    minio_secret_key: S3User.MinIO
+    etcd_root_password: Etcd.Root

deepflow.yml

@@ -0,0 +1,24 @@
+#!/usr/bin/env ansible-playbook
+---
+#==============================================================#
+# File      :   deepflow.yml
+# Desc      :   deploy deepflow server stack
+# Ctime     :   2026-02-04
+# Mtime     :   2026-02-04
+# Path      :   deepflow.yml
+# License   :   Apache-2.0 @ https://pigsty.io/docs/about/license/
+#==============================================================#
+
+- name: DEEPFLOW SERVER
+  become: true
+  hosts: all
+  gather_facts: no
+
+  roles:
+    - { role: node_id         , tags: node-id,        when: deepflow_enabled | default(true) | bool }
+    - { role: deepflow_server , tags: deepflow_server, when: deepflow_enabled | default(true) | bool }
+
+# Usage:
+# 1. Define deepflow group in pigsty.yml
+# 2. Ensure docker is installed: ./docker.yml
+# 3. Run ./deepflow.yml -l <deepflow_group>

roles/deepflow_server/README.md

@@ -0,0 +1,23 @@
+# Role: deepflow_server
+
+Deploy DeepFlow server stack (deepflow-server + deepflow-app + ClickHouse + MinIO)
+with Docker Compose managed by systemd.
+
+## Usage
+
+1. Ensure Docker is installed (`./docker.yml`) and `docker_enabled: true`.
+2. Add hosts to a `deepflow` group with proper vars.
+3. Run `./deepflow.yml -l deepflow`.
+
+## Key Variables
+
+- `deepflow_stack_dir` (default `/opt/deepflow-server`)
+- `deepflow_data` (default `/data/deepflow`)
+- `deepflow_server_grpc_port` (default `20035`)
+- `deepflow_app_port` (default `20880`)
+- `deepflow_s3_access_key`, `deepflow_s3_secret_key`
+
+## Related
+
+- `roles/infra/templates/caddy/Caddyfile` for TLS gRPC ingress
+- `roles/infra/defaults/main.yml` (`deepflow_grpc_*` variables)

roles/deepflow_server/defaults/main.yml

@@ -0,0 +1,31 @@
+---
+#-----------------------------------------------------------------
+# DEEPFLOW SERVER
+#-----------------------------------------------------------------
+deepflow_enabled: true
+
+deepflow_stack_dir: /opt/deepflow-server
+deepflow_data: /data/deepflow
+deepflow_env_file: /etc/default/deepflow-server
+deepflow_compose_file: "{{ deepflow_stack_dir }}/docker-compose.yml"
+
+# images (pin to specific tags before production)
+deepflow_server_image: deepflowio/deepflow-server-ce:latest
+deepflow_app_image: deepflowio/deepflow-app-ce:latest
+deepflow_clickhouse_image: clickhouse/clickhouse-server:24.8
+deepflow_minio_image: minio/minio:RELEASE.2025-01-20T14-49-07Z
+
+# exposed ports
+deepflow_server_grpc_port: 20035
+deepflow_server_http_port: 20417
+deepflow_app_port: 20880
+deepflow_clickhouse_http_port: 18123
+deepflow_clickhouse_tcp_port: 19000
+deepflow_minio_api_port: 19090
+deepflow_minio_console_port: 19091
+
+# S3 backend used by deepflow-server
+deepflow_s3_bucket: deepflow
+deepflow_s3_access_key: deepflow
+deepflow_s3_secret_key: DeepFlow.S3.ChangeMe
+deepflow_s3_region: us-east-1

roles/deepflow_server/meta/main.yml

@@ -0,0 +1,7 @@
+galaxy_info:
+  author: observability.svc.plus
+  description: Deploy DeepFlow stack via Docker Compose
+  license: Apache-2.0
+  min_ansible_version: '2.10'
+
+dependencies: []

roles/deepflow_server/tasks/main.yml

@@ -0,0 +1,86 @@
+---
+#--------------------------------------------------------------#
+# Preflight                                     [deepflow_check]
+#--------------------------------------------------------------#
+- name: check deepflow prerequisites
+  tags: deepflow_check
+  block:
+    - name: assert docker is enabled
+      assert:
+        that:
+          - docker_enabled is defined
+          - docker_enabled | bool
+        fail_msg: "docker_enabled=true is required for deepflow_server role"
+
+    - name: check docker binary exists
+      command: docker --version
+      changed_when: false
+
+#--------------------------------------------------------------#
+# Configure                                      [deepflow_conf]
+#--------------------------------------------------------------#
+- name: configure deepflow-server stack
+  tags: deepflow_conf
+  block:
+    - name: create deepflow directories
+      file:
+        path: "{{ item }}"
+        state: directory
+        owner: root
+        group: root
+        mode: '0755'
+      loop:
+        - "{{ deepflow_stack_dir }}"
+        - "{{ deepflow_data }}"
+        - "{{ deepflow_data }}/clickhouse"
+        - "{{ deepflow_data }}/minio"
+        - "{{ deepflow_data }}/server"
+
+    - name: render deepflow environment
+      template:
+        src: deepflow.env.j2
+        dest: "{{ deepflow_env_file }}"
+        owner: root
+        group: root
+        mode: '0640'
+
+    - name: render deepflow docker compose
+      template:
+        src: docker-compose.yml.j2
+        dest: "{{ deepflow_compose_file }}"
+        owner: root
+        group: root
+        mode: '0644'
+
+    - name: install deepflow systemd unit
+      template:
+        src: deepflow-server.svc
+        dest: "{{ systemd_dir }}/deepflow-server.service"
+        owner: root
+        group: root
+        mode: '0644'
+
+#--------------------------------------------------------------#
+# Launch                                            [deepflow_up]
+#--------------------------------------------------------------#
+- name: launch deepflow-server stack
+  tags: deepflow_up
+  block:
+    - name: start deepflow-server service
+      systemd:
+        name: deepflow-server
+        state: restarted
+        enabled: yes
+        daemon_reload: yes
+
+    - name: wait for deepflow gRPC port
+      wait_for:
+        host: 127.0.0.1
+        port: "{{ deepflow_server_grpc_port }}"
+        timeout: 60
+
+    - name: wait for deepflow app port
+      wait_for:
+        host: 127.0.0.1
+        port: "{{ deepflow_app_port }}"
+        timeout: 60

roles/deepflow_server/templates/deepflow-server.svc

@@ -0,0 +1,17 @@
+[Unit]
+Description=DeepFlow Server Stack (Docker Compose)
+After=docker.service network-online.target
+Wants=docker.service network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+WorkingDirectory={{ deepflow_stack_dir }}
+EnvironmentFile={{ deepflow_env_file }}
+ExecStart=/usr/bin/docker compose --env-file {{ deepflow_env_file }} -f {{ deepflow_compose_file }} up -d
+ExecStop=/usr/bin/docker compose --env-file {{ deepflow_env_file }} -f {{ deepflow_compose_file }} down
+TimeoutStartSec=0
+TimeoutStopSec=120
+
+[Install]
+WantedBy=multi-user.target

roles/deepflow_server/templates/deepflow.env.j2

@@ -0,0 +1,20 @@
+DEEPFLOW_STACK_DIR={{ deepflow_stack_dir }}
+DEEPFLOW_DATA={{ deepflow_data }}
+
+DEEPFLOW_SERVER_IMAGE={{ deepflow_server_image }}
+DEEPFLOW_APP_IMAGE={{ deepflow_app_image }}
+DEEPFLOW_CLICKHOUSE_IMAGE={{ deepflow_clickhouse_image }}
+DEEPFLOW_MINIO_IMAGE={{ deepflow_minio_image }}
+
+DEEPFLOW_SERVER_GRPC_PORT={{ deepflow_server_grpc_port }}
+DEEPFLOW_SERVER_HTTP_PORT={{ deepflow_server_http_port }}
+DEEPFLOW_APP_PORT={{ deepflow_app_port }}
+DEEPFLOW_CLICKHOUSE_HTTP_PORT={{ deepflow_clickhouse_http_port }}
+DEEPFLOW_CLICKHOUSE_TCP_PORT={{ deepflow_clickhouse_tcp_port }}
+DEEPFLOW_MINIO_API_PORT={{ deepflow_minio_api_port }}
+DEEPFLOW_MINIO_CONSOLE_PORT={{ deepflow_minio_console_port }}
+
+DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }}
+DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }}
+DEEPFLOW_S3_SECRET_KEY={{ deepflow_s3_secret_key }}
+DEEPFLOW_S3_REGION={{ deepflow_s3_region }}

roles/deepflow_server/templates/docker-compose.yml.j2

@@ -0,0 +1,64 @@
+version: '3.9'
+
+services:
+  clickhouse:
+    container_name: deepflow-clickhouse
+    image: ${DEEPFLOW_CLICKHOUSE_IMAGE}
+    restart: unless-stopped
+    environment:
+      CLICKHOUSE_DB: deepflow
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
+    ports:
+      - '${DEEPFLOW_CLICKHOUSE_HTTP_PORT}:8123'
+      - '${DEEPFLOW_CLICKHOUSE_TCP_PORT}:9000'
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    volumes:
+      - ${DEEPFLOW_DATA}/clickhouse:/var/lib/clickhouse
+
+  minio:
+    container_name: deepflow-minio
+    image: ${DEEPFLOW_MINIO_IMAGE}
+    restart: unless-stopped
+    environment:
+      MINIO_ROOT_USER: ${DEEPFLOW_S3_ACCESS_KEY}
+      MINIO_ROOT_PASSWORD: ${DEEPFLOW_S3_SECRET_KEY}
+    command: server /data --console-address ':9001'
+    ports:
+      - '${DEEPFLOW_MINIO_API_PORT}:9000'
+      - '${DEEPFLOW_MINIO_CONSOLE_PORT}:9001'
+    volumes:
+      - ${DEEPFLOW_DATA}/minio:/data
+
+  deepflow-server:
+    container_name: deepflow-server
+    image: ${DEEPFLOW_SERVER_IMAGE}
+    restart: unless-stopped
+    depends_on:
+      - clickhouse
+      - minio
+    environment:
+      DEEPFLOW_CLICKHOUSE_ADDR: clickhouse:9000
+      DEEPFLOW_S3_ENDPOINT: http://minio:9000
+      DEEPFLOW_S3_BUCKET: ${DEEPFLOW_S3_BUCKET}
+      DEEPFLOW_S3_ACCESS_KEY: ${DEEPFLOW_S3_ACCESS_KEY}
+      DEEPFLOW_S3_SECRET_KEY: ${DEEPFLOW_S3_SECRET_KEY}
+      DEEPFLOW_S3_REGION: ${DEEPFLOW_S3_REGION}
+    ports:
+      - '${DEEPFLOW_SERVER_GRPC_PORT}:20035'
+      - '${DEEPFLOW_SERVER_HTTP_PORT}:20417'
+    volumes:
+      - ${DEEPFLOW_DATA}/server:/var/lib/deepflow
+
+  deepflow-app:
+    container_name: deepflow-app
+    image: ${DEEPFLOW_APP_IMAGE}
+    restart: unless-stopped
+    depends_on:
+      - deepflow-server
+    environment:
+      DEEPFLOW_SERVER_ADDR: http://deepflow-server:20417
+    ports:
+      - '${DEEPFLOW_APP_PORT}:80'

roles/infra/defaults/main.yml

@@ -38,6 +38,10 @@ infra_extra_services: []          # extra services to be added on infra home pag
 # CADDY vs NGINX
 #-----------------------------------------------------------------
 caddy_enabled: true               # use caddy as ingress controller?
+# Optional DeepFlow agent ingress: TLS gRPC -> Caddy -> deepflow-server(h2c)
+deepflow_grpc_enabled: false      # expose dedicated gRPC ingress for deepflow-agent?
+deepflow_grpc_domain: deepflow-agent.svc.plus
+deepflow_grpc_upstream: 127.0.0.1:20035
 nginx_enabled: false              # enable nginx on this infra node?
 nginx_clean: false                # clean existing nginx config during init?
 nginx_exporter_enabled: true      # enable nginx_exporter on this infra node?

roles/infra/templates/caddy/Caddyfile

@@ -150,3 +150,14 @@
 		file_server
 	}
 }
+
+{% if deepflow_grpc_enabled | default(false) %}
+{{ deepflow_grpc_domain | default('deepflow-agent.svc.plus') }} {
+	# TLS terminates at Caddy, then proxy h2c gRPC to deepflow-server.
+	reverse_proxy {{ deepflow_grpc_upstream | default('127.0.0.1:20035') }} {
+		transport http {
+			versions h2c
+		}
+	}
+}
+{% endif %}