Add lightweight DeepFlow roles and agent modes

README.md

@@ -5,7 +5,7 @@
 
 **Observability.svc.plus** is an observability solution strictly following the Apache 2.0 license.
 
-> **Focus**: Monitoring & Observability (监控/可观测). Integrating OpenTelemetry (OTel), with future plans to incorporate DeepFlow Agent and other open-source NPM (Network Performance Monitoring) probes.
+> **Focus**: Monitoring & Observability (监控/可观测). Integrating OpenTelemetry (OTel), VictoriaMetrics, and DeepFlow-based network observability without long-term raw-flow lock-in.
 
 [Website](https://svc.plus/) | [Public Demo](https://svc.plus/services) | [Blog](https://svc.plus/blogs) | [Support](https://www.svc.plus/support)
 
@@ -141,11 +141,13 @@ This repo now provides dedicated DeepFlow roles:
 - `deepflow_mysql`
 - `deepflow_clickhouse_s3`
 - `deepflow_server`
+- `deepflow_connector`
+- `deepflow_agent`
 
 Quick start:
 
 ```bash
-./configure -c app/deepflow
+./configure -c deepflow/deepflow
 vi pigsty.yml                  # adjust domain/password/ports
 ./deploy.yml
 ./docker.yml
@@ -153,7 +155,16 @@ vi pigsty.yml                  # adjust domain/password/ports
 ./infra.yml -t caddy           # apply deepflow_grpc_domain ingress
 ```
 
-Default inventory template: `conf/app/deepflow.yml`
+Default inventory template: `conf/deepflow/deepflow.yml`
+
+### Lightweight Topology
+
+- `deepflow-server` stays containerized with Docker Compose
+- ClickHouse is kept as short-retention local storage
+- MinIO/S3 is optional in lightweight mode
+- `deepflow_connector` exports selected DeepFlow L4/L7 metrics to VictoriaMetrics
+- `deepflow_agent` supports `binary/systemd`, `docker`, and rendered `k8s` manifests
+- default `deepflow_agent_profile=lite` keeps `pcap` enabled and disables built-in `vector`
 
 ### Remote client example (openclaw.svc.plus)
 
@@ -185,7 +196,7 @@ SSH_SERVER_CLAWBOT_DESCRIPTION=openclaw_server
 
 - **Observability First**: SOTA monitoring for PG / Infra / Node based on VictoriaMetrics, Grafana, and OpenTelemetry.
 - **OTel Integration**: Native support for OpenTelemetry, facilitating unified trace, metric, and log ingestion.
-- **Future Ready**: Planned integration for DeepFlow Agent and other open-source NPM probes for deep network and application observability.
+- **DeepFlow Ready**: Lightweight DeepFlow server/agent deployment with short-lived flow storage and VictoriaMetrics archiving for high-value protocol metrics.
 - **Reliable Base**: Robust self-healing HA clusters, PITR, and secure infrastructure.
 - **Maintainable**: One-Cmd Deploy, IaC support, and easy customization.
 - **Controllable**: Self-sufficient Cloud Neutral FOSS. Run on bare Linux.

conf/deepflow/deepflow.yml

@@ -11,11 +11,11 @@
 #
 #  curl -fsSL https://repo.pigsty.io/get | bash; cd ~/pigsty
 # ./bootstrap                  # prepare local repo & ansible
-# ./configure -c app/deepflow  # use this deepflow config template
+# ./configure -c deepflow/deepflow  # use this deepflow config template
 # vi pigsty.yml                # IMPORTANT: CHANGE CREDENTIALS / DOMAIN
 # ./deploy.yml                 # install infra stack
 # ./docker.yml                 # install docker & docker-compose
-# ./deepflow.yml               # install deepflow with three roles
+# ./deepflow.yml               # install deepflow with compose + optional connector/agent
 
 all:
   children:
@@ -26,6 +26,11 @@ all:
         deepflow_enabled: true
         deepflow_mysql_enabled: true
         deepflow_clickhouse_s3_enabled: true
+        deepflow_connector_enabled: true
+        deepflow_agent_enabled: false
+
+        deepflow_deploy_profile: lite
+        deepflow_storage_mode: short_ttl
 
         deepflow_data: /data/deepflow
 
@@ -39,6 +44,8 @@ all:
         # role: deepflow_clickhouse_s3
         deepflow_clickhouse_http_port: 18123
         deepflow_clickhouse_tcp_port: 19000
+        deepflow_clickhouse_retention_hours: 24
+        deepflow_s3_enabled: false
         deepflow_minio_api_port: 19090
         deepflow_minio_console_port: 19091
         deepflow_s3_bucket: deepflow
@@ -53,6 +60,22 @@ all:
         deepflow_clickhouse_addr: host.docker.internal:19000
         deepflow_s3_endpoint: http://host.docker.internal:19090
         deepflow_mysql_addr: host.docker.internal:13306
+        deepflow_l4_log_ttl_hour: 24
+        deepflow_l7_log_ttl_hour: 24
+        deepflow_flow_metrics_ttl_hour: 24
+        deepflow_metrics_ttl_hour: 24
+        deepflow_prometheus_ttl_hour: 24
+
+        # role: deepflow_connector
+        deepflow_connector_source_endpoint: http://127.0.0.1:20417/metrics
+        deepflow_connector_remote_write_url: http://127.0.0.1:8428/api/v1/write
+
+        # role: deepflow_agent
+        deepflow_agent_mode: binary
+        deepflow_agent_profile: lite
+        deepflow_agent_disable_pcap: false
+        deepflow_agent_disable_vector: true
+        deepflow_agent_grpc_endpoint: "{{ deepflow_grpc_domain }}:443"
 
     infra: { hosts: { 10.10.10.10: { infra_seq: 1 } } }
     etcd:  { hosts: { 10.10.10.10: { etcd_seq: 1 } }, vars: { etcd_cluster: etcd } }

deepflow.yml

@@ -19,6 +19,8 @@
     - { role: deepflow_mysql        , tags: deepflow_mysql,        when: deepflow_mysql_enabled | default(true) | bool }
     - { role: deepflow_clickhouse_s3, tags: deepflow_clickhouse_s3, when: deepflow_clickhouse_s3_enabled | default(true) | bool }
     - { role: deepflow_server       , tags: deepflow_server,       when: deepflow_enabled | default(true) | bool }
+    - { role: deepflow_connector    , tags: deepflow_connector,    when: deepflow_connector_enabled | default(false) | bool }
+    - { role: deepflow_agent        , tags: deepflow_agent,        when: deepflow_agent_enabled | default(false) | bool }
 
 # Usage:
 # 1. Define deepflow group in pigsty.yml

roles/deepflow_agent/README.md

@@ -0,0 +1,27 @@
+# Role: deepflow_agent
+
+Deploy DeepFlow agent in one of three modes:
+
+- `binary + systemd`
+- `docker`
+- `k8s` manifest rendering
+
+## Key Variables
+
+- `deepflow_agent_mode` (`binary`, `docker`, `k8s`)
+- `deepflow_agent_profile` (`lite`, `full`)
+- `deepflow_agent_grpc_endpoint`
+- `deepflow_agent_download_url`
+- `deepflow_agent_binary_path`
+
+## Default Lightweight Profile
+
+The default `lite` profile keeps `pcap` enabled and disables:
+
+- built-in `vector`
+- other optional non-core plugins
+
+## Notes
+
+- `k8s` mode renders a DaemonSet manifest and only applies it when `deepflow_agent_k8s_apply: true`
+- `docker` mode requires `docker_enabled: true`

roles/deepflow_agent/defaults/main.yml

@@ -0,0 +1,41 @@
+---
+#-----------------------------------------------------------------
+# DEEPFLOW AGENT
+#-----------------------------------------------------------------
+deepflow_agent_enabled: false
+deepflow_agent_mode: binary          # binary|docker|k8s
+deepflow_agent_profile: lite         # lite|full
+
+deepflow_agent_stack_dir: /opt/deepflow-agent
+deepflow_agent_env_file: /etc/default/deepflow-agent
+deepflow_agent_compose_file: "{{ deepflow_agent_stack_dir }}/docker-compose.yml"
+deepflow_agent_k8s_file: "{{ deepflow_agent_stack_dir }}/deepflow-agent.yaml"
+deepflow_agent_run_script: /usr/local/bin/run-deepflow-agent.sh
+deepflow_agent_binary_path: /usr/local/bin/deepflow-agent
+deepflow_agent_download_url: ''
+
+deepflow_agent_image: deepflowio/deepflow-agent-ce:latest
+deepflow_agent_grpc_endpoint: "{{ deepflow_grpc_domain | default('deepflow-agent.svc.plus') }}:443"
+deepflow_agent_endpoint_arg: --controller-ips
+deepflow_agent_extra_args: []
+deepflow_agent_disable_pcap: false
+deepflow_agent_disable_vector: true
+deepflow_agent_disable_plugins: true
+deepflow_agent_extra_env: {}
+
+deepflow_agent_host_network: true
+deepflow_agent_container_name: deepflow-agent
+deepflow_agent_k8s_namespace: deepflow
+deepflow_agent_k8s_apply: false
+deepflow_agent_binary_install: true
+deepflow_agent_docker_enabled: true
+
+deepflow_agent_cap_add:
+  - NET_ADMIN
+  - NET_RAW
+  - SYS_ADMIN
+
+deepflow_agent_volume_mounts:
+  - /:/host:ro
+  - /sys:/sys:ro
+  - /var/run/docker.sock:/var/run/docker.sock

roles/deepflow_agent/meta/main.yml

@@ -0,0 +1,7 @@
+galaxy_info:
+  author: observability.svc.plus
+  description: Deploy DeepFlow agent via binary/systemd, Docker, or Kubernetes manifests
+  license: Apache-2.0
+  min_ansible_version: '2.10'
+
+dependencies: []

roles/deepflow_agent/tasks/main.yml

@@ -0,0 +1,147 @@
+---
+#--------------------------------------------------------------#
+# Preflight                                [deepflow_agent_check]
+#--------------------------------------------------------------#
+- name: check deepflow agent mode
+  tags: deepflow_agent_check
+  assert:
+    that:
+      - deepflow_agent_mode in ['binary', 'docker', 'k8s']
+    fail_msg: "deepflow_agent_mode must be one of: binary, docker, k8s"
+
+- name: check deepflow agent grpc endpoint
+  tags: deepflow_agent_check
+  assert:
+    that:
+      - deepflow_agent_grpc_endpoint | default('', true) | length > 0
+    fail_msg: "deepflow_agent_grpc_endpoint is required"
+
+- name: check deepflow agent docker prerequisite
+  tags: deepflow_agent_check
+  when: deepflow_agent_mode == 'docker'
+  block:
+    - name: assert docker is enabled for docker agent mode
+      assert:
+        that:
+          - docker_enabled is defined
+          - docker_enabled | bool
+        fail_msg: "docker_enabled=true is required when deepflow_agent_mode=docker"
+
+    - name: check docker binary exists for docker agent mode
+      command: docker --version
+      changed_when: false
+
+#--------------------------------------------------------------#
+# Configure                                [deepflow_agent_conf]
+#--------------------------------------------------------------#
+- name: create deepflow agent directories
+  tags: deepflow_agent_conf
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+  loop:
+    - "{{ deepflow_agent_stack_dir }}"
+
+- name: render deepflow agent environment
+  tags: deepflow_agent_conf
+  template:
+    src: deepflow-agent.env.j2
+    dest: "{{ deepflow_agent_env_file }}"
+    owner: root
+    group: root
+    mode: '0640'
+
+- name: configure binary deepflow agent
+  tags: deepflow_agent_conf
+  when: deepflow_agent_mode == 'binary'
+  block:
+    - name: download deepflow agent binary when url is provided
+      get_url:
+        url: "{{ deepflow_agent_download_url }}"
+        dest: "{{ deepflow_agent_binary_path }}"
+        mode: '0755'
+      when: deepflow_agent_download_url | default('', true) | length > 0
+
+    - name: verify deepflow agent binary exists
+      stat:
+        path: "{{ deepflow_agent_binary_path }}"
+      register: deepflow_agent_binary_stat
+
+    - name: assert binary path exists
+      assert:
+        that:
+          - deepflow_agent_binary_stat.stat.exists
+        fail_msg: "deepflow_agent_binary_path does not exist. Set deepflow_agent_download_url or provide an existing binary."
+
+    - name: render deepflow agent run script
+      template:
+        src: run-deepflow-agent.sh.j2
+        dest: "{{ deepflow_agent_run_script }}"
+        owner: root
+        group: root
+        mode: '0755'
+
+    - name: install deepflow agent systemd unit
+      template:
+        src: deepflow-agent.svc.j2
+        dest: "{{ systemd_dir }}/deepflow-agent.service"
+        owner: root
+        group: root
+        mode: '0644'
+
+- name: configure docker deepflow agent
+  tags: deepflow_agent_conf
+  when: deepflow_agent_mode == 'docker'
+  block:
+    - name: render docker deepflow agent compose
+      template:
+        src: docker-compose.yml.j2
+        dest: "{{ deepflow_agent_compose_file }}"
+        owner: root
+        group: root
+        mode: '0644'
+
+    - name: install docker deepflow agent systemd unit
+      template:
+        src: deepflow-agent-docker.svc.j2
+        dest: "{{ systemd_dir }}/deepflow-agent.service"
+        owner: root
+        group: root
+        mode: '0644'
+
+- name: configure kubernetes deepflow agent
+  tags: deepflow_agent_conf
+  when: deepflow_agent_mode == 'k8s'
+  block:
+    - name: render deepflow agent kubernetes manifest
+      template:
+        src: deepflow-agent.yaml.j2
+        dest: "{{ deepflow_agent_k8s_file }}"
+        owner: root
+        group: root
+        mode: '0644'
+
+#--------------------------------------------------------------#
+# Launch                                  [deepflow_agent_launch]
+#--------------------------------------------------------------#
+- name: launch binary/docker deepflow agent
+  tags: deepflow_agent_launch
+  when: deepflow_agent_mode in ['binary', 'docker']
+  block:
+    - name: restart deepflow agent systemd service
+      systemd:
+        name: deepflow-agent
+        state: restarted
+        enabled: yes
+        daemon_reload: yes
+
+- name: optionally apply kubernetes manifest
+  tags: deepflow_agent_launch
+  when:
+    - deepflow_agent_mode == 'k8s'
+    - deepflow_agent_k8s_apply | bool
+  command: kubectl apply -f {{ deepflow_agent_k8s_file }}
+  changed_when: true

roles/deepflow_agent/templates/deepflow-agent-docker.svc.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=DeepFlow Agent (Docker)
+After=network-online.target docker.service
+Requires=docker.service
+Wants=network-online.target
+
+[Service]
+WorkingDirectory={{ deepflow_agent_stack_dir }}
+EnvironmentFile={{ deepflow_agent_env_file }}
+ExecStart=/usr/bin/docker compose --env-file {{ deepflow_agent_env_file }} -f {{ deepflow_agent_compose_file }} up -d
+ExecStop=/usr/bin/docker compose --env-file {{ deepflow_agent_env_file }} -f {{ deepflow_agent_compose_file }} down
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

roles/deepflow_agent/templates/deepflow-agent.env.j2

@@ -0,0 +1,12 @@
+DEEPFLOW_AGENT_MODE={{ deepflow_agent_mode }}
+DEEPFLOW_AGENT_PROFILE={{ deepflow_agent_profile }}
+DEEPFLOW_AGENT_BIN={{ deepflow_agent_binary_path }}
+DEEPFLOW_AGENT_ENDPOINT_ARG={{ deepflow_agent_endpoint_arg }}
+DEEPFLOW_GRPC_ENDPOINT={{ deepflow_agent_grpc_endpoint }}
+DEEPFLOW_AGENT_DISABLE_PCAP={{ deepflow_agent_disable_pcap | ternary('true', 'false') }}
+DEEPFLOW_AGENT_DISABLE_VECTOR={{ deepflow_agent_disable_vector | ternary('true', 'false') }}
+DEEPFLOW_AGENT_DISABLE_PLUGINS={{ deepflow_agent_disable_plugins | ternary('true', 'false') }}
+DEEPFLOW_AGENT_ARGS={{ (deepflow_agent_extra_args | default([])) | join(' ') }}
+{% for key, value in (deepflow_agent_extra_env | default({})).items() %}
+{{ key }}={{ value | to_json }}
+{% endfor %}

roles/deepflow_agent/templates/deepflow-agent.svc.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=DeepFlow Agent
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+User=root
+EnvironmentFile={{ deepflow_agent_env_file }}
+ExecStart={{ deepflow_agent_run_script }}
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

roles/deepflow_agent/templates/deepflow-agent.yaml.j2

@@ -0,0 +1,70 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ deepflow_agent_k8s_namespace }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: deepflow-agent
+  namespace: {{ deepflow_agent_k8s_namespace }}
+data:
+  DEEPFLOW_GRPC_ENDPOINT: {{ deepflow_agent_grpc_endpoint | quote }}
+  DEEPFLOW_AGENT_ENDPOINT_ARG: {{ deepflow_agent_endpoint_arg | quote }}
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: deepflow-agent
+  namespace: {{ deepflow_agent_k8s_namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: deepflow-agent
+  template:
+    metadata:
+      labels:
+        app: deepflow-agent
+    spec:
+      hostNetwork: {{ deepflow_agent_host_network | ternary(true, false) }}
+      containers:
+        - name: deepflow-agent
+          image: {{ deepflow_agent_image }}
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - configMapRef:
+                name: deepflow-agent
+          args:
+            - {{ deepflow_agent_endpoint_arg | quote }}
+            - {{ deepflow_agent_grpc_endpoint | quote }}
+{% if deepflow_agent_disable_pcap | bool %}
+            - "--disable-pcap"
+{% endif %}
+{% if deepflow_agent_disable_vector | bool %}
+            - "--disable-vector"
+{% endif %}
+{% if deepflow_agent_disable_plugins | bool %}
+            - "--disable-plugins"
+{% endif %}
+{% for arg in deepflow_agent_extra_args | default([]) %}
+            - {{ arg | quote }}
+{% endfor %}
+          securityContext:
+            privileged: true
+            capabilities:
+              add:
+{% for cap in deepflow_agent_cap_add | default([]) %}
+                - {{ cap }}
+{% endfor %}
+          volumeMounts:
+{% for mount in deepflow_agent_volume_mounts | default([]) %}
+            - name: mount{{ loop.index }}
+              mountPath: {{ mount.split(':')[1] }}
+              readOnly: {{ (mount.split(':') | length > 2 and mount.split(':')[2] == 'ro') | ternary(true, false) }}
+{% endfor %}
+      volumes:
+{% for mount in deepflow_agent_volume_mounts | default([]) %}
+        - name: mount{{ loop.index }}
+          hostPath:
+            path: {{ mount.split(':')[0] }}
+{% endfor %}

roles/deepflow_agent/templates/docker-compose.yml.j2

@@ -0,0 +1,39 @@
+version: '3.9'
+
+services:
+  deepflow-agent:
+    container_name: {{ deepflow_agent_container_name }}
+    image: {{ deepflow_agent_image }}
+    restart: unless-stopped
+{% if deepflow_agent_host_network | bool %}
+    network_mode: host
+{% endif %}
+    privileged: true
+    environment:
+      DEEPFLOW_AGENT_PROFILE: {{ deepflow_agent_profile | to_json }}
+{% for key, value in (deepflow_agent_extra_env | default({})).items() %}
+      {{ key }}: {{ value | to_json }}
+{% endfor %}
+    command:
+      - "{{ deepflow_agent_endpoint_arg }}"
+      - "{{ deepflow_agent_grpc_endpoint }}"
+{% if deepflow_agent_disable_pcap | bool %}
+      - "--disable-pcap"
+{% endif %}
+{% if deepflow_agent_disable_vector | bool %}
+      - "--disable-vector"
+{% endif %}
+{% if deepflow_agent_disable_plugins | bool %}
+      - "--disable-plugins"
+{% endif %}
+{% for arg in deepflow_agent_extra_args | default([]) %}
+      - "{{ arg }}"
+{% endfor %}
+    cap_add:
+{% for cap in deepflow_agent_cap_add | default([]) %}
+      - {{ cap }}
+{% endfor %}
+    volumes:
+{% for mount in deepflow_agent_volume_mounts | default([]) %}
+      - {{ mount }}
+{% endfor %}

roles/deepflow_agent/templates/run-deepflow-agent.sh.j2

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -euo pipefail
+
+. "{{ deepflow_agent_env_file }}"
+
+args=("${DEEPFLOW_AGENT_ENDPOINT_ARG}" "${DEEPFLOW_GRPC_ENDPOINT}")
+
+if [[ "${DEEPFLOW_AGENT_DISABLE_PCAP}" == "true" ]]; then
+  args+=("--disable-pcap")
+fi
+if [[ "${DEEPFLOW_AGENT_DISABLE_VECTOR}" == "true" ]]; then
+  args+=("--disable-vector")
+fi
+if [[ "${DEEPFLOW_AGENT_DISABLE_PLUGINS}" == "true" ]]; then
+  args+=("--disable-plugins")
+fi
+if [[ -n "${DEEPFLOW_AGENT_ARGS}" ]]; then
+  # shellcheck disable=SC2206
+  extra_args=(${DEEPFLOW_AGENT_ARGS})
+  args+=("${extra_args[@]}")
+fi
+
+exec "{{ deepflow_agent_binary_path }}" "${args[@]}"

roles/deepflow_clickhouse_s3/README.md

@@ -1,6 +1,9 @@
 # Role: deepflow_clickhouse_s3
 
-Deploy ClickHouse + MinIO(S3) backend for DeepFlow with Docker Compose managed by systemd.
+Deploy ClickHouse backend for DeepFlow with Docker Compose managed by systemd.
+
+The default layout is optimized for short-term DeepFlow storage. MinIO/S3 can be disabled when the
+deployment only needs local short-retention ClickHouse.
 
 ## Key Variables
 
@@ -8,3 +11,5 @@ Deploy ClickHouse + MinIO(S3) backend for DeepFlow with Docker Compose managed b
 - `deepflow_clickhouse_http_port` (default `18123`)
 - `deepflow_minio_api_port` (default `19090`)
 - `deepflow_s3_access_key` / `deepflow_s3_secret_key`
+- `deepflow_clickhouse_retention_hours` (default `24`)
+- `deepflow_s3_enabled` (default `true`)

roles/deepflow_clickhouse_s3/defaults/main.yml

@@ -3,10 +3,12 @@
 # DEEPFLOW CLICKHOUSE + S3
 #-----------------------------------------------------------------
 deepflow_clickhouse_s3_enabled: true
+deepflow_storage_mode: short_ttl
 
 deepflow_clickhouse_s3_stack_dir: /opt/deepflow-clickhouse-s3
 deepflow_clickhouse_s3_env_file: /etc/default/deepflow-clickhouse-s3
 deepflow_clickhouse_s3_compose_file: "{{ deepflow_clickhouse_s3_stack_dir }}/docker-compose.yml"
+deepflow_clickhouse_config_dir: "{{ deepflow_clickhouse_s3_stack_dir }}/clickhouse-config.d"
 
 deepflow_data: /data/deepflow
 deepflow_clickhouse_data: "{{ deepflow_data }}/clickhouse"
@@ -19,6 +21,8 @@ deepflow_clickhouse_http_port: 18123
 deepflow_clickhouse_tcp_port: 19000
 deepflow_minio_api_port: 19090
 deepflow_minio_console_port: 19091
+deepflow_clickhouse_retention_hours: 24
+deepflow_s3_enabled: true
 
 deepflow_s3_bucket: deepflow
 deepflow_s3_access_key: deepflow

roles/deepflow_clickhouse_s3/tasks/main.yml

@@ -33,7 +33,9 @@
         - "{{ deepflow_clickhouse_s3_stack_dir }}"
         - "{{ deepflow_data }}"
         - "{{ deepflow_clickhouse_data }}"
+        - "{{ deepflow_clickhouse_config_dir }}"
         - "{{ deepflow_s3_data }}"
+      when: item != deepflow_s3_data or deepflow_s3_enabled | bool
 
     - name: render deepflow clickhouse+s3 environment
       template:
@@ -43,6 +45,14 @@
         group: root
         mode: '0640'
 
+    - name: render deepflow clickhouse config
+      template:
+        src: clickhouse-config.d/retention.xml.j2
+        dest: "{{ deepflow_clickhouse_config_dir }}/retention.xml"
+        owner: root
+        group: root
+        mode: '0644'
+
     - name: render deepflow clickhouse+s3 docker compose
       template:
         src: docker-compose.yml.j2
@@ -83,3 +93,4 @@
         host: 127.0.0.1
         port: "{{ deepflow_minio_api_port }}"
         timeout: 60
+      when: deepflow_s3_enabled | bool

roles/deepflow_clickhouse_s3/templates/clickhouse-config.d/retention.xml.j2

@@ -0,0 +1,13 @@
+<clickhouse>
+  <logger>
+    <level>information</level>
+  </logger>
+  <profiles>
+    <default>
+      <max_execution_time>60</max_execution_time>
+    </default>
+  </profiles>
+  <!-- DeepFlow retention is enforced from server.yaml and documented here for operator visibility. -->
+  <!-- deepflow_clickhouse_retention_hours={{ deepflow_clickhouse_retention_hours }} -->
+  <!-- deepflow_storage_mode={{ deepflow_storage_mode }} -->
+</clickhouse>

roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.env.j2

@@ -6,8 +6,11 @@ DEEPFLOW_S3_DATA={{ deepflow_s3_data }}
 
 DEEPFLOW_CLICKHOUSE_HTTP_PORT={{ deepflow_clickhouse_http_port }}
 DEEPFLOW_CLICKHOUSE_TCP_PORT={{ deepflow_clickhouse_tcp_port }}
+DEEPFLOW_CLICKHOUSE_CONFIG_DIR={{ deepflow_clickhouse_config_dir }}
 DEEPFLOW_MINIO_API_PORT={{ deepflow_minio_api_port }}
 DEEPFLOW_MINIO_CONSOLE_PORT={{ deepflow_minio_console_port }}
+DEEPFLOW_CLICKHOUSE_RETENTION_HOURS={{ deepflow_clickhouse_retention_hours }}
+DEEPFLOW_S3_ENABLED={{ deepflow_s3_enabled | ternary('true', 'false') }}
 
 DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }}
 DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }}

roles/deepflow_clickhouse_s3/templates/docker-compose.yml.j2

@@ -17,7 +17,9 @@ services:
         hard: 262144
     volumes:
       - ${DEEPFLOW_CLICKHOUSE_DATA}:/var/lib/clickhouse
+      - ${DEEPFLOW_CLICKHOUSE_CONFIG_DIR}:/etc/clickhouse-server/config.d:ro
 
+{% if deepflow_s3_enabled | bool %}
   minio:
     container_name: deepflow-minio
     image: ${DEEPFLOW_MINIO_IMAGE}
@@ -31,3 +33,4 @@ services:
       - '${DEEPFLOW_MINIO_CONSOLE_PORT}:9001'
     volumes:
       - ${DEEPFLOW_S3_DATA}:/data
+{% endif %}

roles/deepflow_connector/README.md

@@ -0,0 +1,17 @@
+# Role: deepflow_connector
+
+Deploy a lightweight OpenTelemetry Collector bridge that scrapes DeepFlow metrics and writes the
+selected L4/L7 protocol metrics into VictoriaMetrics.
+
+## Key Variables
+
+- `deepflow_connector_source_endpoint`
+- `deepflow_connector_metric_keep_regex`
+- `deepflow_connector_remote_write_url`
+- `deepflow_connector_scrape_interval`
+
+## Scope
+
+- Supports metrics export only
+- Does not export protocol logs
+- Does not export traces

roles/deepflow_connector/defaults/main.yml

@@ -0,0 +1,23 @@
+---
+#-----------------------------------------------------------------
+# DEEPFLOW CONNECTOR
+#-----------------------------------------------------------------
+deepflow_connector_enabled: false
+deepflow_connector_mode: docker
+
+deepflow_connector_stack_dir: /opt/deepflow-connector
+deepflow_connector_env_file: /etc/default/deepflow-connector
+deepflow_connector_compose_file: "{{ deepflow_connector_stack_dir }}/docker-compose.yml"
+deepflow_connector_config_file: "{{ deepflow_connector_stack_dir }}/otel-collector.yaml"
+
+deepflow_connector_image: otel/opentelemetry-collector-contrib:0.121.0
+deepflow_connector_container_name: deepflow-connector
+deepflow_connector_listen_port: 19091
+
+deepflow_connector_source_endpoint: http://127.0.0.1:20417/metrics
+deepflow_connector_source_job_name: deepflow
+deepflow_connector_metrics_profile: l4_l7
+deepflow_connector_metric_keep_regex: '^(deepflow_.*|flow_.*|l4_.*|l7_.*)$'
+deepflow_connector_scrape_interval: 30s
+deepflow_connector_remote_write_url: http://127.0.0.1:8428/api/v1/write
+deepflow_connector_remote_write_headers: {}

roles/deepflow_connector/meta/main.yml

@@ -0,0 +1,7 @@
+galaxy_info:
+  author: observability.svc.plus
+  description: Export DeepFlow L4/L7 metrics to VictoriaMetrics through OpenTelemetry Collector
+  license: Apache-2.0
+  min_ansible_version: '2.10'
+
+dependencies: []

roles/deepflow_connector/tasks/main.yml

@@ -0,0 +1,84 @@
+---
+#--------------------------------------------------------------#
+# Preflight                            [deepflow_connector_check]
+#--------------------------------------------------------------#
+- name: check deepflow connector prerequisites
+  tags: deepflow_connector_check
+  block:
+    - name: assert docker is enabled
+      assert:
+        that:
+          - docker_enabled is defined
+          - docker_enabled | bool
+        fail_msg: "docker_enabled=true is required for deepflow_connector"
+
+    - name: check docker binary exists
+      command: docker --version
+      changed_when: false
+
+#--------------------------------------------------------------#
+# Configure                             [deepflow_connector_conf]
+#--------------------------------------------------------------#
+- name: configure deepflow connector stack
+  tags: deepflow_connector_conf
+  block:
+    - name: create deepflow connector directories
+      file:
+        path: "{{ item }}"
+        state: directory
+        owner: root
+        group: root
+        mode: '0755'
+      loop:
+        - "{{ deepflow_connector_stack_dir }}"
+
+    - name: render deepflow connector environment
+      template:
+        src: deepflow-connector.env.j2
+        dest: "{{ deepflow_connector_env_file }}"
+        owner: root
+        group: root
+        mode: '0640'
+
+    - name: render deepflow connector collector config
+      template:
+        src: otel-collector.yaml.j2
+        dest: "{{ deepflow_connector_config_file }}"
+        owner: root
+        group: root
+        mode: '0644'
+
+    - name: render deepflow connector docker compose
+      template:
+        src: docker-compose.yml.j2
+        dest: "{{ deepflow_connector_compose_file }}"
+        owner: root
+        group: root
+        mode: '0644'
+
+    - name: install deepflow connector systemd unit
+      template:
+        src: deepflow-connector.svc.j2
+        dest: "{{ systemd_dir }}/deepflow-connector.service"
+        owner: root
+        group: root
+        mode: '0644'
+
+#--------------------------------------------------------------#
+# Launch                               [deepflow_connector_launch]
+#--------------------------------------------------------------#
+- name: launch deepflow connector stack
+  tags: deepflow_connector_launch
+  block:
+    - name: restart deepflow connector service
+      systemd:
+        name: deepflow-connector
+        state: restarted
+        enabled: yes
+        daemon_reload: yes
+
+    - name: wait for deepflow connector service online
+      wait_for:
+        host: 127.0.0.1
+        port: "{{ deepflow_connector_listen_port }}"
+        timeout: 60

roles/deepflow_connector/templates/deepflow-connector.env.j2

@@ -0,0 +1,6 @@
+DEEPFLOW_CONNECTOR_IMAGE={{ deepflow_connector_image }}
+DEEPFLOW_CONNECTOR_CONFIG_FILE={{ deepflow_connector_config_file }}
+DEEPFLOW_CONNECTOR_LISTEN_PORT={{ deepflow_connector_listen_port }}
+DEEPFLOW_CONNECTOR_SOURCE_ENDPOINT={{ deepflow_connector_source_endpoint }}
+DEEPFLOW_CONNECTOR_SOURCE_JOB_NAME={{ deepflow_connector_source_job_name }}
+DEEPFLOW_CONNECTOR_REMOTE_WRITE_URL={{ deepflow_connector_remote_write_url }}

roles/deepflow_connector/templates/deepflow-connector.svc.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=DeepFlow Connector
+After=network-online.target docker.service
+Requires=docker.service
+Wants=network-online.target
+
+[Service]
+WorkingDirectory={{ deepflow_connector_stack_dir }}
+EnvironmentFile={{ deepflow_connector_env_file }}
+ExecStart=/usr/bin/docker compose --env-file {{ deepflow_connector_env_file }} -f {{ deepflow_connector_compose_file }} up -d
+ExecStop=/usr/bin/docker compose --env-file {{ deepflow_connector_env_file }} -f {{ deepflow_connector_compose_file }} down
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

roles/deepflow_connector/templates/docker-compose.yml.j2

@@ -0,0 +1,13 @@
+version: '3.9'
+
+services:
+  deepflow-connector:
+    container_name: {{ deepflow_connector_container_name }}
+    image: ${DEEPFLOW_CONNECTOR_IMAGE}
+    restart: unless-stopped
+    command:
+      - --config=/etc/otelcol-contrib/config.yaml
+    ports:
+      - '${DEEPFLOW_CONNECTOR_LISTEN_PORT}:13133'
+    volumes:
+      - ${DEEPFLOW_CONNECTOR_CONFIG_FILE}:/etc/otelcol-contrib/config.yaml:ro

roles/deepflow_connector/templates/otel-collector.yaml.j2

@@ -0,0 +1,41 @@
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+
+receivers:
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: {{ deepflow_connector_source_job_name | to_json }}
+          scrape_interval: {{ deepflow_connector_scrape_interval }}
+          static_configs:
+            - targets:
+                - {{ deepflow_connector_source_endpoint | regex_replace('^https?://', '') | regex_replace('/.*$', '') | to_json }}
+          metrics_path: {{ ('/' + (deepflow_connector_source_endpoint | regex_replace('^https?://[^/]+', '') | regex_replace('^$', '/metrics') | regex_replace('^//', '/'))) | to_json }}
+
+processors:
+  filter/deepflow:
+    metrics:
+      include:
+        match_type: regexp
+        metric_names:
+          - {{ deepflow_connector_metric_keep_regex | to_json }}
+  batch: {}
+
+exporters:
+  prometheusremotewrite:
+    endpoint: {{ deepflow_connector_remote_write_url | to_json }}
+{% if deepflow_connector_remote_write_headers %}
+    headers:
+{% for key, value in (deepflow_connector_remote_write_headers | default({})).items() %}
+      {{ key }}: {{ value | to_json }}
+{% endfor %}
+{% endif %}
+
+service:
+  extensions: [health_check]
+  pipelines:
+    metrics:
+      receivers: [prometheus]
+      processors: [filter/deepflow, batch]
+      exporters: [prometheusremotewrite]

roles/deepflow_server/README.md

@@ -2,11 +2,18 @@
 
 Deploy DeepFlow control plane (`deepflow-server` + `deepflow-app`) with Docker Compose managed by systemd.
 
+This role is intentionally container-only. It does not provide a host binary install path for
+`deepflow-server`.
+
 This role expects backend dependencies from separate roles:
 
 - `deepflow_mysql`
 - `deepflow_clickhouse_s3`
 
+Optional downstream integration:
+
+- `deepflow_connector`
+
 ## Usage
 
 1. Ensure Docker is installed (`./docker.yml`) and `docker_enabled: true`.
@@ -19,3 +26,12 @@ This role expects backend dependencies from separate roles:
 - `deepflow_app_port` (default `20880`)
 - `deepflow_clickhouse_addr` (default `host.docker.internal:19000`)
 - `deepflow_s3_endpoint` (default `http://host.docker.internal:19090`)
+- `deepflow_clickhouse_retention_hours` (default `24`)
+- `deepflow_storage_mode` (default `short_ttl`)
+
+## Lightweight Defaults
+
+- `deepflow_deploy_profile: lite`
+- `deepflow_storage_mode: short_ttl`
+- retention is written to DeepFlow `server.yaml` in hours
+- S3/MinIO is optional and can be disabled with `deepflow_s3_enabled: false`

roles/deepflow_server/defaults/main.yml

@@ -3,11 +3,15 @@
 # DEEPFLOW SERVER
 #-----------------------------------------------------------------
 deepflow_enabled: true
+deepflow_deploy_profile: lite
+deepflow_storage_mode: short_ttl
 
 deepflow_stack_dir: /opt/deepflow-server
 deepflow_data: /data/deepflow
 deepflow_env_file: /etc/default/deepflow-server
 deepflow_compose_file: "{{ deepflow_stack_dir }}/docker-compose.yml"
+deepflow_server_config_dir: "{{ deepflow_stack_dir }}/server.yaml.d"
+deepflow_server_config_file: "{{ deepflow_server_config_dir }}/server.yaml"
 
 # images (pin to specific tags before production)
 deepflow_server_image: deepflowio/deepflow-server-ce:latest
@@ -20,13 +24,28 @@ deepflow_app_port: 20880
 
 # backend endpoints (provided by dedicated roles)
 deepflow_clickhouse_addr: host.docker.internal:19000
+deepflow_clickhouse_database: deepflow
 deepflow_s3_endpoint: http://host.docker.internal:19090
 deepflow_s3_bucket: deepflow
 deepflow_s3_access_key: deepflow
 deepflow_s3_secret_key: DeepFlow.S3.ChangeMe
 deepflow_s3_region: us-east-1
+deepflow_s3_enabled: true
 
 deepflow_mysql_addr: host.docker.internal:13306
 deepflow_mysql_user: deepflow
 deepflow_mysql_password: DeepFlow.MySQL.ChangeMe
 deepflow_mysql_database: deepflow
+
+# Lightweight retention handled by DeepFlow server config.
+deepflow_clickhouse_retention_hours: 24
+deepflow_l4_log_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}"
+deepflow_l7_log_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}"
+deepflow_flow_metrics_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}"
+deepflow_metrics_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}"
+deepflow_prometheus_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}"
+
+# Optional server config overrides.
+deepflow_server_listen_ip: 0.0.0.0
+deepflow_server_extra_env: {}
+deepflow_server_extra_labels: {}

roles/deepflow_server/tasks/main.yml

@@ -33,6 +33,7 @@
         - "{{ deepflow_stack_dir }}"
         - "{{ deepflow_data }}"
         - "{{ deepflow_data }}/server"
+        - "{{ deepflow_server_config_dir }}"
 
     - name: render deepflow environment
       template:
@@ -42,6 +43,14 @@
         group: root
         mode: '0640'
 
+    - name: render deepflow server config
+      template:
+        src: server.yaml.j2
+        dest: "{{ deepflow_server_config_file }}"
+        owner: root
+        group: root
+        mode: '0644'
+
     - name: render deepflow docker compose
       template:
         src: docker-compose.yml.j2

roles/deepflow_server/templates/deepflow.env.j2

@@ -7,13 +7,16 @@ DEEPFLOW_APP_IMAGE={{ deepflow_app_image }}
 DEEPFLOW_SERVER_GRPC_PORT={{ deepflow_server_grpc_port }}
 DEEPFLOW_SERVER_HTTP_PORT={{ deepflow_server_http_port }}
 DEEPFLOW_APP_PORT={{ deepflow_app_port }}
+DEEPFLOW_SERVER_CONFIG_FILE={{ deepflow_server_config_file }}
 
 DEEPFLOW_CLICKHOUSE_ADDR={{ deepflow_clickhouse_addr }}
+DEEPFLOW_CLICKHOUSE_DATABASE={{ deepflow_clickhouse_database }}
 DEEPFLOW_S3_ENDPOINT={{ deepflow_s3_endpoint }}
 DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }}
 DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }}
 DEEPFLOW_S3_SECRET_KEY={{ deepflow_s3_secret_key }}
 DEEPFLOW_S3_REGION={{ deepflow_s3_region }}
+DEEPFLOW_S3_ENABLED={{ deepflow_s3_enabled | ternary('true', 'false') }}
 
 DEEPFLOW_MYSQL_ADDR={{ deepflow_mysql_addr }}
 DEEPFLOW_MYSQL_USER={{ deepflow_mysql_user }}

roles/deepflow_server/templates/docker-compose.yml.j2

@@ -18,11 +18,22 @@ services:
       DEEPFLOW_MYSQL_USER: ${DEEPFLOW_MYSQL_USER}
       DEEPFLOW_MYSQL_PASSWORD: ${DEEPFLOW_MYSQL_PASSWORD}
       DEEPFLOW_MYSQL_DATABASE: ${DEEPFLOW_MYSQL_DATABASE}
+{% for key, value in (deepflow_server_extra_env | default({})).items() %}
+      {{ key }}: {{ value | to_json }}
+{% endfor %}
     ports:
       - '${DEEPFLOW_SERVER_GRPC_PORT}:20035'
       - '${DEEPFLOW_SERVER_HTTP_PORT}:20417'
     volumes:
       - ${DEEPFLOW_DATA}/server:/var/lib/deepflow
+      - ${DEEPFLOW_SERVER_CONFIG_FILE}:/etc/deepflow/server.yaml:ro
+      - ${DEEPFLOW_SERVER_CONFIG_FILE}:/etc/server.yaml:ro
+{% if deepflow_server_extra_labels | default({}) %}
+    labels:
+{% for key, value in (deepflow_server_extra_labels | default({})).items() %}
+      {{ key }}: {{ value | to_json }}
+{% endfor %}
+{% endif %}
 
   deepflow-app:
     container_name: deepflow-app

roles/deepflow_server/templates/server.yaml.j2

@@ -0,0 +1,27 @@
+spec:
+  listen-port: {{ deepflow_server_http_port }}
+  listen-node-port: {{ deepflow_server_grpc_port }}
+  listen-node-ip: {{ deepflow_server_listen_ip | quote }}
+mysql:
+  host: {{ deepflow_mysql_addr.split(':')[0] | quote }}
+  port: {{ (deepflow_mysql_addr.split(':') | length > 1) | ternary(deepflow_mysql_addr.split(':')[1], '3306') }}
+  user: {{ deepflow_mysql_user | quote }}
+  password: {{ deepflow_mysql_password | quote }}
+  database: {{ deepflow_mysql_database | quote }}
+clickhouse:
+  host: {{ deepflow_clickhouse_addr.split(':')[0] | quote }}
+  port: {{ (deepflow_clickhouse_addr.split(':') | length > 1) | ternary(deepflow_clickhouse_addr.split(':')[1], '9000') }}
+  database: {{ deepflow_clickhouse_database | quote }}
+  flow-metrics-ttl-hour: {{ deepflow_flow_metrics_ttl_hour }}
+  metrics-ttl-hour: {{ deepflow_metrics_ttl_hour }}
+  l4-log-ttl-hour: {{ deepflow_l4_log_ttl_hour }}
+  l7-log-ttl-hour: {{ deepflow_l7_log_ttl_hour }}
+  prometheus-ttl-hour: {{ deepflow_prometheus_ttl_hour }}
+{% if deepflow_s3_enabled | bool %}
+s3:
+  endpoint: {{ deepflow_s3_endpoint | quote }}
+  bucket: {{ deepflow_s3_bucket | quote }}
+  ak: {{ deepflow_s3_access_key | quote }}
+  sk: {{ deepflow_s3_secret_key | quote }}
+  region: {{ deepflow_s3_region | quote }}
+{% endif %}