From 8a780487d18874c76fbd9d34ae78a71e5b004e03 Mon Sep 17 00:00:00 2001 From: Haitao Pan Date: Tue, 17 Mar 2026 08:16:32 +0800 Subject: [PATCH] Add lightweight DeepFlow roles and agent modes --- README.md | 19 ++- conf/deepflow/deepflow.yml | 27 +++- deepflow.yml | 2 + roles/deepflow_agent/README.md | 27 ++++ roles/deepflow_agent/defaults/main.yml | 41 +++++ roles/deepflow_agent/meta/main.yml | 7 + roles/deepflow_agent/tasks/main.yml | 147 ++++++++++++++++++ .../templates/deepflow-agent-docker.svc.j2 | 15 ++ .../templates/deepflow-agent.env.j2 | 12 ++ .../templates/deepflow-agent.svc.j2 | 14 ++ .../templates/deepflow-agent.yaml.j2 | 70 +++++++++ .../templates/docker-compose.yml.j2 | 39 +++++ .../templates/run-deepflow-agent.sh.j2 | 23 +++ roles/deepflow_clickhouse_s3/README.md | 7 +- .../deepflow_clickhouse_s3/defaults/main.yml | 4 + roles/deepflow_clickhouse_s3/tasks/main.yml | 11 ++ .../clickhouse-config.d/retention.xml.j2 | 13 ++ .../templates/deepflow-clickhouse-s3.env.j2 | 3 + .../templates/docker-compose.yml.j2 | 3 + roles/deepflow_connector/README.md | 17 ++ roles/deepflow_connector/defaults/main.yml | 23 +++ roles/deepflow_connector/meta/main.yml | 7 + roles/deepflow_connector/tasks/main.yml | 84 ++++++++++ .../templates/deepflow-connector.env.j2 | 6 + .../templates/deepflow-connector.svc.j2 | 15 ++ .../templates/docker-compose.yml.j2 | 13 ++ .../templates/otel-collector.yaml.j2 | 41 +++++ roles/deepflow_server/README.md | 16 ++ roles/deepflow_server/defaults/main.yml | 19 +++ roles/deepflow_server/tasks/main.yml | 9 ++ .../deepflow_server/templates/deepflow.env.j2 | 3 + .../templates/docker-compose.yml.j2 | 11 ++ .../deepflow_server/templates/server.yaml.j2 | 27 ++++ 33 files changed, 768 insertions(+), 7 deletions(-) create mode 100644 roles/deepflow_agent/README.md create mode 100644 roles/deepflow_agent/defaults/main.yml create mode 100644 roles/deepflow_agent/meta/main.yml create mode 100644 roles/deepflow_agent/tasks/main.yml create mode 100644 roles/deepflow_agent/templates/deepflow-agent-docker.svc.j2 create mode 100644 roles/deepflow_agent/templates/deepflow-agent.env.j2 create mode 100644 roles/deepflow_agent/templates/deepflow-agent.svc.j2 create mode 100644 roles/deepflow_agent/templates/deepflow-agent.yaml.j2 create mode 100644 roles/deepflow_agent/templates/docker-compose.yml.j2 create mode 100644 roles/deepflow_agent/templates/run-deepflow-agent.sh.j2 create mode 100644 roles/deepflow_clickhouse_s3/templates/clickhouse-config.d/retention.xml.j2 create mode 100644 roles/deepflow_connector/README.md create mode 100644 roles/deepflow_connector/defaults/main.yml create mode 100644 roles/deepflow_connector/meta/main.yml create mode 100644 roles/deepflow_connector/tasks/main.yml create mode 100644 roles/deepflow_connector/templates/deepflow-connector.env.j2 create mode 100644 roles/deepflow_connector/templates/deepflow-connector.svc.j2 create mode 100644 roles/deepflow_connector/templates/docker-compose.yml.j2 create mode 100644 roles/deepflow_connector/templates/otel-collector.yaml.j2 create mode 100644 roles/deepflow_server/templates/server.yaml.j2 diff --git a/README.md b/README.md index 2b43648..e2ad73c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ **Observability.svc.plus** is an observability solution strictly following the Apache 2.0 license. -> **Focus**: Monitoring & Observability (监控/可观测). Integrating OpenTelemetry (OTel), with future plans to incorporate DeepFlow Agent and other open-source NPM (Network Performance Monitoring) probes. +> **Focus**: Monitoring & Observability (监控/可观测). Integrating OpenTelemetry (OTel), VictoriaMetrics, and DeepFlow-based network observability without long-term raw-flow lock-in. [Website](https://svc.plus/) | [Public Demo](https://svc.plus/services) | [Blog](https://svc.plus/blogs) | [Support](https://www.svc.plus/support) @@ -141,11 +141,13 @@ This repo now provides dedicated DeepFlow roles: - `deepflow_mysql` - `deepflow_clickhouse_s3` - `deepflow_server` +- `deepflow_connector` +- `deepflow_agent` Quick start: ```bash -./configure -c app/deepflow +./configure -c deepflow/deepflow vi pigsty.yml # adjust domain/password/ports ./deploy.yml ./docker.yml @@ -153,7 +155,16 @@ vi pigsty.yml # adjust domain/password/ports ./infra.yml -t caddy # apply deepflow_grpc_domain ingress ``` -Default inventory template: `conf/app/deepflow.yml` +Default inventory template: `conf/deepflow/deepflow.yml` + +### Lightweight Topology + +- `deepflow-server` stays containerized with Docker Compose +- ClickHouse is kept as short-retention local storage +- MinIO/S3 is optional in lightweight mode +- `deepflow_connector` exports selected DeepFlow L4/L7 metrics to VictoriaMetrics +- `deepflow_agent` supports `binary/systemd`, `docker`, and rendered `k8s` manifests +- default `deepflow_agent_profile=lite` keeps `pcap` enabled and disables built-in `vector` ### Remote client example (openclaw.svc.plus) @@ -185,7 +196,7 @@ SSH_SERVER_CLAWBOT_DESCRIPTION=openclaw_server - **Observability First**: SOTA monitoring for PG / Infra / Node based on VictoriaMetrics, Grafana, and OpenTelemetry. - **OTel Integration**: Native support for OpenTelemetry, facilitating unified trace, metric, and log ingestion. -- **Future Ready**: Planned integration for DeepFlow Agent and other open-source NPM probes for deep network and application observability. +- **DeepFlow Ready**: Lightweight DeepFlow server/agent deployment with short-lived flow storage and VictoriaMetrics archiving for high-value protocol metrics. - **Reliable Base**: Robust self-healing HA clusters, PITR, and secure infrastructure. - **Maintainable**: One-Cmd Deploy, IaC support, and easy customization. - **Controllable**: Self-sufficient Cloud Neutral FOSS. Run on bare Linux. diff --git a/conf/deepflow/deepflow.yml b/conf/deepflow/deepflow.yml index 563e31d..4059974 100644 --- a/conf/deepflow/deepflow.yml +++ b/conf/deepflow/deepflow.yml @@ -11,11 +11,11 @@ # # curl -fsSL https://repo.pigsty.io/get | bash; cd ~/pigsty # ./bootstrap # prepare local repo & ansible -# ./configure -c app/deepflow # use this deepflow config template +# ./configure -c deepflow/deepflow # use this deepflow config template # vi pigsty.yml # IMPORTANT: CHANGE CREDENTIALS / DOMAIN # ./deploy.yml # install infra stack # ./docker.yml # install docker & docker-compose -# ./deepflow.yml # install deepflow with three roles +# ./deepflow.yml # install deepflow with compose + optional connector/agent all: children: @@ -26,6 +26,11 @@ all: deepflow_enabled: true deepflow_mysql_enabled: true deepflow_clickhouse_s3_enabled: true + deepflow_connector_enabled: true + deepflow_agent_enabled: false + + deepflow_deploy_profile: lite + deepflow_storage_mode: short_ttl deepflow_data: /data/deepflow @@ -39,6 +44,8 @@ all: # role: deepflow_clickhouse_s3 deepflow_clickhouse_http_port: 18123 deepflow_clickhouse_tcp_port: 19000 + deepflow_clickhouse_retention_hours: 24 + deepflow_s3_enabled: false deepflow_minio_api_port: 19090 deepflow_minio_console_port: 19091 deepflow_s3_bucket: deepflow @@ -53,6 +60,22 @@ all: deepflow_clickhouse_addr: host.docker.internal:19000 deepflow_s3_endpoint: http://host.docker.internal:19090 deepflow_mysql_addr: host.docker.internal:13306 + deepflow_l4_log_ttl_hour: 24 + deepflow_l7_log_ttl_hour: 24 + deepflow_flow_metrics_ttl_hour: 24 + deepflow_metrics_ttl_hour: 24 + deepflow_prometheus_ttl_hour: 24 + + # role: deepflow_connector + deepflow_connector_source_endpoint: http://127.0.0.1:20417/metrics + deepflow_connector_remote_write_url: http://127.0.0.1:8428/api/v1/write + + # role: deepflow_agent + deepflow_agent_mode: binary + deepflow_agent_profile: lite + deepflow_agent_disable_pcap: false + deepflow_agent_disable_vector: true + deepflow_agent_grpc_endpoint: "{{ deepflow_grpc_domain }}:443" infra: { hosts: { 10.10.10.10: { infra_seq: 1 } } } etcd: { hosts: { 10.10.10.10: { etcd_seq: 1 } }, vars: { etcd_cluster: etcd } } diff --git a/deepflow.yml b/deepflow.yml index 80b88c8..7b90c6f 100644 --- a/deepflow.yml +++ b/deepflow.yml @@ -19,6 +19,8 @@ - { role: deepflow_mysql , tags: deepflow_mysql, when: deepflow_mysql_enabled | default(true) | bool } - { role: deepflow_clickhouse_s3, tags: deepflow_clickhouse_s3, when: deepflow_clickhouse_s3_enabled | default(true) | bool } - { role: deepflow_server , tags: deepflow_server, when: deepflow_enabled | default(true) | bool } + - { role: deepflow_connector , tags: deepflow_connector, when: deepflow_connector_enabled | default(false) | bool } + - { role: deepflow_agent , tags: deepflow_agent, when: deepflow_agent_enabled | default(false) | bool } # Usage: # 1. Define deepflow group in pigsty.yml diff --git a/roles/deepflow_agent/README.md b/roles/deepflow_agent/README.md new file mode 100644 index 0000000..923ecef --- /dev/null +++ b/roles/deepflow_agent/README.md @@ -0,0 +1,27 @@ +# Role: deepflow_agent + +Deploy DeepFlow agent in one of three modes: + +- `binary + systemd` +- `docker` +- `k8s` manifest rendering + +## Key Variables + +- `deepflow_agent_mode` (`binary`, `docker`, `k8s`) +- `deepflow_agent_profile` (`lite`, `full`) +- `deepflow_agent_grpc_endpoint` +- `deepflow_agent_download_url` +- `deepflow_agent_binary_path` + +## Default Lightweight Profile + +The default `lite` profile keeps `pcap` enabled and disables: + +- built-in `vector` +- other optional non-core plugins + +## Notes + +- `k8s` mode renders a DaemonSet manifest and only applies it when `deepflow_agent_k8s_apply: true` +- `docker` mode requires `docker_enabled: true` diff --git a/roles/deepflow_agent/defaults/main.yml b/roles/deepflow_agent/defaults/main.yml new file mode 100644 index 0000000..3cedbae --- /dev/null +++ b/roles/deepflow_agent/defaults/main.yml @@ -0,0 +1,41 @@ +--- +#----------------------------------------------------------------- +# DEEPFLOW AGENT +#----------------------------------------------------------------- +deepflow_agent_enabled: false +deepflow_agent_mode: binary # binary|docker|k8s +deepflow_agent_profile: lite # lite|full + +deepflow_agent_stack_dir: /opt/deepflow-agent +deepflow_agent_env_file: /etc/default/deepflow-agent +deepflow_agent_compose_file: "{{ deepflow_agent_stack_dir }}/docker-compose.yml" +deepflow_agent_k8s_file: "{{ deepflow_agent_stack_dir }}/deepflow-agent.yaml" +deepflow_agent_run_script: /usr/local/bin/run-deepflow-agent.sh +deepflow_agent_binary_path: /usr/local/bin/deepflow-agent +deepflow_agent_download_url: '' + +deepflow_agent_image: deepflowio/deepflow-agent-ce:latest +deepflow_agent_grpc_endpoint: "{{ deepflow_grpc_domain | default('deepflow-agent.svc.plus') }}:443" +deepflow_agent_endpoint_arg: --controller-ips +deepflow_agent_extra_args: [] +deepflow_agent_disable_pcap: false +deepflow_agent_disable_vector: true +deepflow_agent_disable_plugins: true +deepflow_agent_extra_env: {} + +deepflow_agent_host_network: true +deepflow_agent_container_name: deepflow-agent +deepflow_agent_k8s_namespace: deepflow +deepflow_agent_k8s_apply: false +deepflow_agent_binary_install: true +deepflow_agent_docker_enabled: true + +deepflow_agent_cap_add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + +deepflow_agent_volume_mounts: + - /:/host:ro + - /sys:/sys:ro + - /var/run/docker.sock:/var/run/docker.sock diff --git a/roles/deepflow_agent/meta/main.yml b/roles/deepflow_agent/meta/main.yml new file mode 100644 index 0000000..000026a --- /dev/null +++ b/roles/deepflow_agent/meta/main.yml @@ -0,0 +1,7 @@ +galaxy_info: + author: observability.svc.plus + description: Deploy DeepFlow agent via binary/systemd, Docker, or Kubernetes manifests + license: Apache-2.0 + min_ansible_version: '2.10' + +dependencies: [] diff --git a/roles/deepflow_agent/tasks/main.yml b/roles/deepflow_agent/tasks/main.yml new file mode 100644 index 0000000..c6d48ee --- /dev/null +++ b/roles/deepflow_agent/tasks/main.yml @@ -0,0 +1,147 @@ +--- +#--------------------------------------------------------------# +# Preflight [deepflow_agent_check] +#--------------------------------------------------------------# +- name: check deepflow agent mode + tags: deepflow_agent_check + assert: + that: + - deepflow_agent_mode in ['binary', 'docker', 'k8s'] + fail_msg: "deepflow_agent_mode must be one of: binary, docker, k8s" + +- name: check deepflow agent grpc endpoint + tags: deepflow_agent_check + assert: + that: + - deepflow_agent_grpc_endpoint | default('', true) | length > 0 + fail_msg: "deepflow_agent_grpc_endpoint is required" + +- name: check deepflow agent docker prerequisite + tags: deepflow_agent_check + when: deepflow_agent_mode == 'docker' + block: + - name: assert docker is enabled for docker agent mode + assert: + that: + - docker_enabled is defined + - docker_enabled | bool + fail_msg: "docker_enabled=true is required when deepflow_agent_mode=docker" + + - name: check docker binary exists for docker agent mode + command: docker --version + changed_when: false + +#--------------------------------------------------------------# +# Configure [deepflow_agent_conf] +#--------------------------------------------------------------# +- name: create deepflow agent directories + tags: deepflow_agent_conf + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - "{{ deepflow_agent_stack_dir }}" + +- name: render deepflow agent environment + tags: deepflow_agent_conf + template: + src: deepflow-agent.env.j2 + dest: "{{ deepflow_agent_env_file }}" + owner: root + group: root + mode: '0640' + +- name: configure binary deepflow agent + tags: deepflow_agent_conf + when: deepflow_agent_mode == 'binary' + block: + - name: download deepflow agent binary when url is provided + get_url: + url: "{{ deepflow_agent_download_url }}" + dest: "{{ deepflow_agent_binary_path }}" + mode: '0755' + when: deepflow_agent_download_url | default('', true) | length > 0 + + - name: verify deepflow agent binary exists + stat: + path: "{{ deepflow_agent_binary_path }}" + register: deepflow_agent_binary_stat + + - name: assert binary path exists + assert: + that: + - deepflow_agent_binary_stat.stat.exists + fail_msg: "deepflow_agent_binary_path does not exist. Set deepflow_agent_download_url or provide an existing binary." + + - name: render deepflow agent run script + template: + src: run-deepflow-agent.sh.j2 + dest: "{{ deepflow_agent_run_script }}" + owner: root + group: root + mode: '0755' + + - name: install deepflow agent systemd unit + template: + src: deepflow-agent.svc.j2 + dest: "{{ systemd_dir }}/deepflow-agent.service" + owner: root + group: root + mode: '0644' + +- name: configure docker deepflow agent + tags: deepflow_agent_conf + when: deepflow_agent_mode == 'docker' + block: + - name: render docker deepflow agent compose + template: + src: docker-compose.yml.j2 + dest: "{{ deepflow_agent_compose_file }}" + owner: root + group: root + mode: '0644' + + - name: install docker deepflow agent systemd unit + template: + src: deepflow-agent-docker.svc.j2 + dest: "{{ systemd_dir }}/deepflow-agent.service" + owner: root + group: root + mode: '0644' + +- name: configure kubernetes deepflow agent + tags: deepflow_agent_conf + when: deepflow_agent_mode == 'k8s' + block: + - name: render deepflow agent kubernetes manifest + template: + src: deepflow-agent.yaml.j2 + dest: "{{ deepflow_agent_k8s_file }}" + owner: root + group: root + mode: '0644' + +#--------------------------------------------------------------# +# Launch [deepflow_agent_launch] +#--------------------------------------------------------------# +- name: launch binary/docker deepflow agent + tags: deepflow_agent_launch + when: deepflow_agent_mode in ['binary', 'docker'] + block: + - name: restart deepflow agent systemd service + systemd: + name: deepflow-agent + state: restarted + enabled: yes + daemon_reload: yes + +- name: optionally apply kubernetes manifest + tags: deepflow_agent_launch + when: + - deepflow_agent_mode == 'k8s' + - deepflow_agent_k8s_apply | bool + command: kubectl apply -f {{ deepflow_agent_k8s_file }} + changed_when: true diff --git a/roles/deepflow_agent/templates/deepflow-agent-docker.svc.j2 b/roles/deepflow_agent/templates/deepflow-agent-docker.svc.j2 new file mode 100644 index 0000000..2035a1e --- /dev/null +++ b/roles/deepflow_agent/templates/deepflow-agent-docker.svc.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=DeepFlow Agent (Docker) +After=network-online.target docker.service +Requires=docker.service +Wants=network-online.target + +[Service] +WorkingDirectory={{ deepflow_agent_stack_dir }} +EnvironmentFile={{ deepflow_agent_env_file }} +ExecStart=/usr/bin/docker compose --env-file {{ deepflow_agent_env_file }} -f {{ deepflow_agent_compose_file }} up -d +ExecStop=/usr/bin/docker compose --env-file {{ deepflow_agent_env_file }} -f {{ deepflow_agent_compose_file }} down +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/roles/deepflow_agent/templates/deepflow-agent.env.j2 b/roles/deepflow_agent/templates/deepflow-agent.env.j2 new file mode 100644 index 0000000..dca82ee --- /dev/null +++ b/roles/deepflow_agent/templates/deepflow-agent.env.j2 @@ -0,0 +1,12 @@ +DEEPFLOW_AGENT_MODE={{ deepflow_agent_mode }} +DEEPFLOW_AGENT_PROFILE={{ deepflow_agent_profile }} +DEEPFLOW_AGENT_BIN={{ deepflow_agent_binary_path }} +DEEPFLOW_AGENT_ENDPOINT_ARG={{ deepflow_agent_endpoint_arg }} +DEEPFLOW_GRPC_ENDPOINT={{ deepflow_agent_grpc_endpoint }} +DEEPFLOW_AGENT_DISABLE_PCAP={{ deepflow_agent_disable_pcap | ternary('true', 'false') }} +DEEPFLOW_AGENT_DISABLE_VECTOR={{ deepflow_agent_disable_vector | ternary('true', 'false') }} +DEEPFLOW_AGENT_DISABLE_PLUGINS={{ deepflow_agent_disable_plugins | ternary('true', 'false') }} +DEEPFLOW_AGENT_ARGS={{ (deepflow_agent_extra_args | default([])) | join(' ') }} +{% for key, value in (deepflow_agent_extra_env | default({})).items() %} +{{ key }}={{ value | to_json }} +{% endfor %} diff --git a/roles/deepflow_agent/templates/deepflow-agent.svc.j2 b/roles/deepflow_agent/templates/deepflow-agent.svc.j2 new file mode 100644 index 0000000..0a6e78b --- /dev/null +++ b/roles/deepflow_agent/templates/deepflow-agent.svc.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=DeepFlow Agent +After=network-online.target +Wants=network-online.target + +[Service] +User=root +EnvironmentFile={{ deepflow_agent_env_file }} +ExecStart={{ deepflow_agent_run_script }} +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/roles/deepflow_agent/templates/deepflow-agent.yaml.j2 b/roles/deepflow_agent/templates/deepflow-agent.yaml.j2 new file mode 100644 index 0000000..37f61dd --- /dev/null +++ b/roles/deepflow_agent/templates/deepflow-agent.yaml.j2 @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ deepflow_agent_k8s_namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: deepflow-agent + namespace: {{ deepflow_agent_k8s_namespace }} +data: + DEEPFLOW_GRPC_ENDPOINT: {{ deepflow_agent_grpc_endpoint | quote }} + DEEPFLOW_AGENT_ENDPOINT_ARG: {{ deepflow_agent_endpoint_arg | quote }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: deepflow-agent + namespace: {{ deepflow_agent_k8s_namespace }} +spec: + selector: + matchLabels: + app: deepflow-agent + template: + metadata: + labels: + app: deepflow-agent + spec: + hostNetwork: {{ deepflow_agent_host_network | ternary(true, false) }} + containers: + - name: deepflow-agent + image: {{ deepflow_agent_image }} + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: deepflow-agent + args: + - {{ deepflow_agent_endpoint_arg | quote }} + - {{ deepflow_agent_grpc_endpoint | quote }} +{% if deepflow_agent_disable_pcap | bool %} + - "--disable-pcap" +{% endif %} +{% if deepflow_agent_disable_vector | bool %} + - "--disable-vector" +{% endif %} +{% if deepflow_agent_disable_plugins | bool %} + - "--disable-plugins" +{% endif %} +{% for arg in deepflow_agent_extra_args | default([]) %} + - {{ arg | quote }} +{% endfor %} + securityContext: + privileged: true + capabilities: + add: +{% for cap in deepflow_agent_cap_add | default([]) %} + - {{ cap }} +{% endfor %} + volumeMounts: +{% for mount in deepflow_agent_volume_mounts | default([]) %} + - name: mount{{ loop.index }} + mountPath: {{ mount.split(':')[1] }} + readOnly: {{ (mount.split(':') | length > 2 and mount.split(':')[2] == 'ro') | ternary(true, false) }} +{% endfor %} + volumes: +{% for mount in deepflow_agent_volume_mounts | default([]) %} + - name: mount{{ loop.index }} + hostPath: + path: {{ mount.split(':')[0] }} +{% endfor %} diff --git a/roles/deepflow_agent/templates/docker-compose.yml.j2 b/roles/deepflow_agent/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..f7e57c6 --- /dev/null +++ b/roles/deepflow_agent/templates/docker-compose.yml.j2 @@ -0,0 +1,39 @@ +version: '3.9' + +services: + deepflow-agent: + container_name: {{ deepflow_agent_container_name }} + image: {{ deepflow_agent_image }} + restart: unless-stopped +{% if deepflow_agent_host_network | bool %} + network_mode: host +{% endif %} + privileged: true + environment: + DEEPFLOW_AGENT_PROFILE: {{ deepflow_agent_profile | to_json }} +{% for key, value in (deepflow_agent_extra_env | default({})).items() %} + {{ key }}: {{ value | to_json }} +{% endfor %} + command: + - "{{ deepflow_agent_endpoint_arg }}" + - "{{ deepflow_agent_grpc_endpoint }}" +{% if deepflow_agent_disable_pcap | bool %} + - "--disable-pcap" +{% endif %} +{% if deepflow_agent_disable_vector | bool %} + - "--disable-vector" +{% endif %} +{% if deepflow_agent_disable_plugins | bool %} + - "--disable-plugins" +{% endif %} +{% for arg in deepflow_agent_extra_args | default([]) %} + - "{{ arg }}" +{% endfor %} + cap_add: +{% for cap in deepflow_agent_cap_add | default([]) %} + - {{ cap }} +{% endfor %} + volumes: +{% for mount in deepflow_agent_volume_mounts | default([]) %} + - {{ mount }} +{% endfor %} diff --git a/roles/deepflow_agent/templates/run-deepflow-agent.sh.j2 b/roles/deepflow_agent/templates/run-deepflow-agent.sh.j2 new file mode 100644 index 0000000..531d8a6 --- /dev/null +++ b/roles/deepflow_agent/templates/run-deepflow-agent.sh.j2 @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail + +. "{{ deepflow_agent_env_file }}" + +args=("${DEEPFLOW_AGENT_ENDPOINT_ARG}" "${DEEPFLOW_GRPC_ENDPOINT}") + +if [[ "${DEEPFLOW_AGENT_DISABLE_PCAP}" == "true" ]]; then + args+=("--disable-pcap") +fi +if [[ "${DEEPFLOW_AGENT_DISABLE_VECTOR}" == "true" ]]; then + args+=("--disable-vector") +fi +if [[ "${DEEPFLOW_AGENT_DISABLE_PLUGINS}" == "true" ]]; then + args+=("--disable-plugins") +fi +if [[ -n "${DEEPFLOW_AGENT_ARGS}" ]]; then + # shellcheck disable=SC2206 + extra_args=(${DEEPFLOW_AGENT_ARGS}) + args+=("${extra_args[@]}") +fi + +exec "{{ deepflow_agent_binary_path }}" "${args[@]}" diff --git a/roles/deepflow_clickhouse_s3/README.md b/roles/deepflow_clickhouse_s3/README.md index 19e4f1a..4e94740 100644 --- a/roles/deepflow_clickhouse_s3/README.md +++ b/roles/deepflow_clickhouse_s3/README.md @@ -1,6 +1,9 @@ # Role: deepflow_clickhouse_s3 -Deploy ClickHouse + MinIO(S3) backend for DeepFlow with Docker Compose managed by systemd. +Deploy ClickHouse backend for DeepFlow with Docker Compose managed by systemd. + +The default layout is optimized for short-term DeepFlow storage. MinIO/S3 can be disabled when the +deployment only needs local short-retention ClickHouse. ## Key Variables @@ -8,3 +11,5 @@ Deploy ClickHouse + MinIO(S3) backend for DeepFlow with Docker Compose managed b - `deepflow_clickhouse_http_port` (default `18123`) - `deepflow_minio_api_port` (default `19090`) - `deepflow_s3_access_key` / `deepflow_s3_secret_key` +- `deepflow_clickhouse_retention_hours` (default `24`) +- `deepflow_s3_enabled` (default `true`) diff --git a/roles/deepflow_clickhouse_s3/defaults/main.yml b/roles/deepflow_clickhouse_s3/defaults/main.yml index 301a48c..26a8d7f 100644 --- a/roles/deepflow_clickhouse_s3/defaults/main.yml +++ b/roles/deepflow_clickhouse_s3/defaults/main.yml @@ -3,10 +3,12 @@ # DEEPFLOW CLICKHOUSE + S3 #----------------------------------------------------------------- deepflow_clickhouse_s3_enabled: true +deepflow_storage_mode: short_ttl deepflow_clickhouse_s3_stack_dir: /opt/deepflow-clickhouse-s3 deepflow_clickhouse_s3_env_file: /etc/default/deepflow-clickhouse-s3 deepflow_clickhouse_s3_compose_file: "{{ deepflow_clickhouse_s3_stack_dir }}/docker-compose.yml" +deepflow_clickhouse_config_dir: "{{ deepflow_clickhouse_s3_stack_dir }}/clickhouse-config.d" deepflow_data: /data/deepflow deepflow_clickhouse_data: "{{ deepflow_data }}/clickhouse" @@ -19,6 +21,8 @@ deepflow_clickhouse_http_port: 18123 deepflow_clickhouse_tcp_port: 19000 deepflow_minio_api_port: 19090 deepflow_minio_console_port: 19091 +deepflow_clickhouse_retention_hours: 24 +deepflow_s3_enabled: true deepflow_s3_bucket: deepflow deepflow_s3_access_key: deepflow diff --git a/roles/deepflow_clickhouse_s3/tasks/main.yml b/roles/deepflow_clickhouse_s3/tasks/main.yml index 5a867c6..2697725 100644 --- a/roles/deepflow_clickhouse_s3/tasks/main.yml +++ b/roles/deepflow_clickhouse_s3/tasks/main.yml @@ -33,7 +33,9 @@ - "{{ deepflow_clickhouse_s3_stack_dir }}" - "{{ deepflow_data }}" - "{{ deepflow_clickhouse_data }}" + - "{{ deepflow_clickhouse_config_dir }}" - "{{ deepflow_s3_data }}" + when: item != deepflow_s3_data or deepflow_s3_enabled | bool - name: render deepflow clickhouse+s3 environment template: @@ -43,6 +45,14 @@ group: root mode: '0640' + - name: render deepflow clickhouse config + template: + src: clickhouse-config.d/retention.xml.j2 + dest: "{{ deepflow_clickhouse_config_dir }}/retention.xml" + owner: root + group: root + mode: '0644' + - name: render deepflow clickhouse+s3 docker compose template: src: docker-compose.yml.j2 @@ -83,3 +93,4 @@ host: 127.0.0.1 port: "{{ deepflow_minio_api_port }}" timeout: 60 + when: deepflow_s3_enabled | bool diff --git a/roles/deepflow_clickhouse_s3/templates/clickhouse-config.d/retention.xml.j2 b/roles/deepflow_clickhouse_s3/templates/clickhouse-config.d/retention.xml.j2 new file mode 100644 index 0000000..b35c490 --- /dev/null +++ b/roles/deepflow_clickhouse_s3/templates/clickhouse-config.d/retention.xml.j2 @@ -0,0 +1,13 @@ + + + information + + + + 60 + + + + + + diff --git a/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.env.j2 b/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.env.j2 index a946bef..3b80026 100644 --- a/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.env.j2 +++ b/roles/deepflow_clickhouse_s3/templates/deepflow-clickhouse-s3.env.j2 @@ -6,8 +6,11 @@ DEEPFLOW_S3_DATA={{ deepflow_s3_data }} DEEPFLOW_CLICKHOUSE_HTTP_PORT={{ deepflow_clickhouse_http_port }} DEEPFLOW_CLICKHOUSE_TCP_PORT={{ deepflow_clickhouse_tcp_port }} +DEEPFLOW_CLICKHOUSE_CONFIG_DIR={{ deepflow_clickhouse_config_dir }} DEEPFLOW_MINIO_API_PORT={{ deepflow_minio_api_port }} DEEPFLOW_MINIO_CONSOLE_PORT={{ deepflow_minio_console_port }} +DEEPFLOW_CLICKHOUSE_RETENTION_HOURS={{ deepflow_clickhouse_retention_hours }} +DEEPFLOW_S3_ENABLED={{ deepflow_s3_enabled | ternary('true', 'false') }} DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }} DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }} diff --git a/roles/deepflow_clickhouse_s3/templates/docker-compose.yml.j2 b/roles/deepflow_clickhouse_s3/templates/docker-compose.yml.j2 index edd7d6b..5abac9c 100644 --- a/roles/deepflow_clickhouse_s3/templates/docker-compose.yml.j2 +++ b/roles/deepflow_clickhouse_s3/templates/docker-compose.yml.j2 @@ -17,7 +17,9 @@ services: hard: 262144 volumes: - ${DEEPFLOW_CLICKHOUSE_DATA}:/var/lib/clickhouse + - ${DEEPFLOW_CLICKHOUSE_CONFIG_DIR}:/etc/clickhouse-server/config.d:ro +{% if deepflow_s3_enabled | bool %} minio: container_name: deepflow-minio image: ${DEEPFLOW_MINIO_IMAGE} @@ -31,3 +33,4 @@ services: - '${DEEPFLOW_MINIO_CONSOLE_PORT}:9001' volumes: - ${DEEPFLOW_S3_DATA}:/data +{% endif %} diff --git a/roles/deepflow_connector/README.md b/roles/deepflow_connector/README.md new file mode 100644 index 0000000..176c6a3 --- /dev/null +++ b/roles/deepflow_connector/README.md @@ -0,0 +1,17 @@ +# Role: deepflow_connector + +Deploy a lightweight OpenTelemetry Collector bridge that scrapes DeepFlow metrics and writes the +selected L4/L7 protocol metrics into VictoriaMetrics. + +## Key Variables + +- `deepflow_connector_source_endpoint` +- `deepflow_connector_metric_keep_regex` +- `deepflow_connector_remote_write_url` +- `deepflow_connector_scrape_interval` + +## Scope + +- Supports metrics export only +- Does not export protocol logs +- Does not export traces diff --git a/roles/deepflow_connector/defaults/main.yml b/roles/deepflow_connector/defaults/main.yml new file mode 100644 index 0000000..b530ebf --- /dev/null +++ b/roles/deepflow_connector/defaults/main.yml @@ -0,0 +1,23 @@ +--- +#----------------------------------------------------------------- +# DEEPFLOW CONNECTOR +#----------------------------------------------------------------- +deepflow_connector_enabled: false +deepflow_connector_mode: docker + +deepflow_connector_stack_dir: /opt/deepflow-connector +deepflow_connector_env_file: /etc/default/deepflow-connector +deepflow_connector_compose_file: "{{ deepflow_connector_stack_dir }}/docker-compose.yml" +deepflow_connector_config_file: "{{ deepflow_connector_stack_dir }}/otel-collector.yaml" + +deepflow_connector_image: otel/opentelemetry-collector-contrib:0.121.0 +deepflow_connector_container_name: deepflow-connector +deepflow_connector_listen_port: 19091 + +deepflow_connector_source_endpoint: http://127.0.0.1:20417/metrics +deepflow_connector_source_job_name: deepflow +deepflow_connector_metrics_profile: l4_l7 +deepflow_connector_metric_keep_regex: '^(deepflow_.*|flow_.*|l4_.*|l7_.*)$' +deepflow_connector_scrape_interval: 30s +deepflow_connector_remote_write_url: http://127.0.0.1:8428/api/v1/write +deepflow_connector_remote_write_headers: {} diff --git a/roles/deepflow_connector/meta/main.yml b/roles/deepflow_connector/meta/main.yml new file mode 100644 index 0000000..a1ac62e --- /dev/null +++ b/roles/deepflow_connector/meta/main.yml @@ -0,0 +1,7 @@ +galaxy_info: + author: observability.svc.plus + description: Export DeepFlow L4/L7 metrics to VictoriaMetrics through OpenTelemetry Collector + license: Apache-2.0 + min_ansible_version: '2.10' + +dependencies: [] diff --git a/roles/deepflow_connector/tasks/main.yml b/roles/deepflow_connector/tasks/main.yml new file mode 100644 index 0000000..82be160 --- /dev/null +++ b/roles/deepflow_connector/tasks/main.yml @@ -0,0 +1,84 @@ +--- +#--------------------------------------------------------------# +# Preflight [deepflow_connector_check] +#--------------------------------------------------------------# +- name: check deepflow connector prerequisites + tags: deepflow_connector_check + block: + - name: assert docker is enabled + assert: + that: + - docker_enabled is defined + - docker_enabled | bool + fail_msg: "docker_enabled=true is required for deepflow_connector" + + - name: check docker binary exists + command: docker --version + changed_when: false + +#--------------------------------------------------------------# +# Configure [deepflow_connector_conf] +#--------------------------------------------------------------# +- name: configure deepflow connector stack + tags: deepflow_connector_conf + block: + - name: create deepflow connector directories + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - "{{ deepflow_connector_stack_dir }}" + + - name: render deepflow connector environment + template: + src: deepflow-connector.env.j2 + dest: "{{ deepflow_connector_env_file }}" + owner: root + group: root + mode: '0640' + + - name: render deepflow connector collector config + template: + src: otel-collector.yaml.j2 + dest: "{{ deepflow_connector_config_file }}" + owner: root + group: root + mode: '0644' + + - name: render deepflow connector docker compose + template: + src: docker-compose.yml.j2 + dest: "{{ deepflow_connector_compose_file }}" + owner: root + group: root + mode: '0644' + + - name: install deepflow connector systemd unit + template: + src: deepflow-connector.svc.j2 + dest: "{{ systemd_dir }}/deepflow-connector.service" + owner: root + group: root + mode: '0644' + +#--------------------------------------------------------------# +# Launch [deepflow_connector_launch] +#--------------------------------------------------------------# +- name: launch deepflow connector stack + tags: deepflow_connector_launch + block: + - name: restart deepflow connector service + systemd: + name: deepflow-connector + state: restarted + enabled: yes + daemon_reload: yes + + - name: wait for deepflow connector service online + wait_for: + host: 127.0.0.1 + port: "{{ deepflow_connector_listen_port }}" + timeout: 60 diff --git a/roles/deepflow_connector/templates/deepflow-connector.env.j2 b/roles/deepflow_connector/templates/deepflow-connector.env.j2 new file mode 100644 index 0000000..a655ee5 --- /dev/null +++ b/roles/deepflow_connector/templates/deepflow-connector.env.j2 @@ -0,0 +1,6 @@ +DEEPFLOW_CONNECTOR_IMAGE={{ deepflow_connector_image }} +DEEPFLOW_CONNECTOR_CONFIG_FILE={{ deepflow_connector_config_file }} +DEEPFLOW_CONNECTOR_LISTEN_PORT={{ deepflow_connector_listen_port }} +DEEPFLOW_CONNECTOR_SOURCE_ENDPOINT={{ deepflow_connector_source_endpoint }} +DEEPFLOW_CONNECTOR_SOURCE_JOB_NAME={{ deepflow_connector_source_job_name }} +DEEPFLOW_CONNECTOR_REMOTE_WRITE_URL={{ deepflow_connector_remote_write_url }} diff --git a/roles/deepflow_connector/templates/deepflow-connector.svc.j2 b/roles/deepflow_connector/templates/deepflow-connector.svc.j2 new file mode 100644 index 0000000..1ea29ac --- /dev/null +++ b/roles/deepflow_connector/templates/deepflow-connector.svc.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=DeepFlow Connector +After=network-online.target docker.service +Requires=docker.service +Wants=network-online.target + +[Service] +WorkingDirectory={{ deepflow_connector_stack_dir }} +EnvironmentFile={{ deepflow_connector_env_file }} +ExecStart=/usr/bin/docker compose --env-file {{ deepflow_connector_env_file }} -f {{ deepflow_connector_compose_file }} up -d +ExecStop=/usr/bin/docker compose --env-file {{ deepflow_connector_env_file }} -f {{ deepflow_connector_compose_file }} down +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/roles/deepflow_connector/templates/docker-compose.yml.j2 b/roles/deepflow_connector/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..9ee6d30 --- /dev/null +++ b/roles/deepflow_connector/templates/docker-compose.yml.j2 @@ -0,0 +1,13 @@ +version: '3.9' + +services: + deepflow-connector: + container_name: {{ deepflow_connector_container_name }} + image: ${DEEPFLOW_CONNECTOR_IMAGE} + restart: unless-stopped + command: + - --config=/etc/otelcol-contrib/config.yaml + ports: + - '${DEEPFLOW_CONNECTOR_LISTEN_PORT}:13133' + volumes: + - ${DEEPFLOW_CONNECTOR_CONFIG_FILE}:/etc/otelcol-contrib/config.yaml:ro diff --git a/roles/deepflow_connector/templates/otel-collector.yaml.j2 b/roles/deepflow_connector/templates/otel-collector.yaml.j2 new file mode 100644 index 0000000..ec6af5f --- /dev/null +++ b/roles/deepflow_connector/templates/otel-collector.yaml.j2 @@ -0,0 +1,41 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + prometheus: + config: + scrape_configs: + - job_name: {{ deepflow_connector_source_job_name | to_json }} + scrape_interval: {{ deepflow_connector_scrape_interval }} + static_configs: + - targets: + - {{ deepflow_connector_source_endpoint | regex_replace('^https?://', '') | regex_replace('/.*$', '') | to_json }} + metrics_path: {{ ('/' + (deepflow_connector_source_endpoint | regex_replace('^https?://[^/]+', '') | regex_replace('^$', '/metrics') | regex_replace('^//', '/'))) | to_json }} + +processors: + filter/deepflow: + metrics: + include: + match_type: regexp + metric_names: + - {{ deepflow_connector_metric_keep_regex | to_json }} + batch: {} + +exporters: + prometheusremotewrite: + endpoint: {{ deepflow_connector_remote_write_url | to_json }} +{% if deepflow_connector_remote_write_headers %} + headers: +{% for key, value in (deepflow_connector_remote_write_headers | default({})).items() %} + {{ key }}: {{ value | to_json }} +{% endfor %} +{% endif %} + +service: + extensions: [health_check] + pipelines: + metrics: + receivers: [prometheus] + processors: [filter/deepflow, batch] + exporters: [prometheusremotewrite] diff --git a/roles/deepflow_server/README.md b/roles/deepflow_server/README.md index 0a2f2a0..107cdbc 100644 --- a/roles/deepflow_server/README.md +++ b/roles/deepflow_server/README.md @@ -2,11 +2,18 @@ Deploy DeepFlow control plane (`deepflow-server` + `deepflow-app`) with Docker Compose managed by systemd. +This role is intentionally container-only. It does not provide a host binary install path for +`deepflow-server`. + This role expects backend dependencies from separate roles: - `deepflow_mysql` - `deepflow_clickhouse_s3` +Optional downstream integration: + +- `deepflow_connector` + ## Usage 1. Ensure Docker is installed (`./docker.yml`) and `docker_enabled: true`. @@ -19,3 +26,12 @@ This role expects backend dependencies from separate roles: - `deepflow_app_port` (default `20880`) - `deepflow_clickhouse_addr` (default `host.docker.internal:19000`) - `deepflow_s3_endpoint` (default `http://host.docker.internal:19090`) +- `deepflow_clickhouse_retention_hours` (default `24`) +- `deepflow_storage_mode` (default `short_ttl`) + +## Lightweight Defaults + +- `deepflow_deploy_profile: lite` +- `deepflow_storage_mode: short_ttl` +- retention is written to DeepFlow `server.yaml` in hours +- S3/MinIO is optional and can be disabled with `deepflow_s3_enabled: false` diff --git a/roles/deepflow_server/defaults/main.yml b/roles/deepflow_server/defaults/main.yml index d70832a..4ca4430 100644 --- a/roles/deepflow_server/defaults/main.yml +++ b/roles/deepflow_server/defaults/main.yml @@ -3,11 +3,15 @@ # DEEPFLOW SERVER #----------------------------------------------------------------- deepflow_enabled: true +deepflow_deploy_profile: lite +deepflow_storage_mode: short_ttl deepflow_stack_dir: /opt/deepflow-server deepflow_data: /data/deepflow deepflow_env_file: /etc/default/deepflow-server deepflow_compose_file: "{{ deepflow_stack_dir }}/docker-compose.yml" +deepflow_server_config_dir: "{{ deepflow_stack_dir }}/server.yaml.d" +deepflow_server_config_file: "{{ deepflow_server_config_dir }}/server.yaml" # images (pin to specific tags before production) deepflow_server_image: deepflowio/deepflow-server-ce:latest @@ -20,13 +24,28 @@ deepflow_app_port: 20880 # backend endpoints (provided by dedicated roles) deepflow_clickhouse_addr: host.docker.internal:19000 +deepflow_clickhouse_database: deepflow deepflow_s3_endpoint: http://host.docker.internal:19090 deepflow_s3_bucket: deepflow deepflow_s3_access_key: deepflow deepflow_s3_secret_key: DeepFlow.S3.ChangeMe deepflow_s3_region: us-east-1 +deepflow_s3_enabled: true deepflow_mysql_addr: host.docker.internal:13306 deepflow_mysql_user: deepflow deepflow_mysql_password: DeepFlow.MySQL.ChangeMe deepflow_mysql_database: deepflow + +# Lightweight retention handled by DeepFlow server config. +deepflow_clickhouse_retention_hours: 24 +deepflow_l4_log_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}" +deepflow_l7_log_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}" +deepflow_flow_metrics_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}" +deepflow_metrics_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}" +deepflow_prometheus_ttl_hour: "{{ deepflow_clickhouse_retention_hours }}" + +# Optional server config overrides. +deepflow_server_listen_ip: 0.0.0.0 +deepflow_server_extra_env: {} +deepflow_server_extra_labels: {} diff --git a/roles/deepflow_server/tasks/main.yml b/roles/deepflow_server/tasks/main.yml index 3de7620..2f4289d 100644 --- a/roles/deepflow_server/tasks/main.yml +++ b/roles/deepflow_server/tasks/main.yml @@ -33,6 +33,7 @@ - "{{ deepflow_stack_dir }}" - "{{ deepflow_data }}" - "{{ deepflow_data }}/server" + - "{{ deepflow_server_config_dir }}" - name: render deepflow environment template: @@ -42,6 +43,14 @@ group: root mode: '0640' + - name: render deepflow server config + template: + src: server.yaml.j2 + dest: "{{ deepflow_server_config_file }}" + owner: root + group: root + mode: '0644' + - name: render deepflow docker compose template: src: docker-compose.yml.j2 diff --git a/roles/deepflow_server/templates/deepflow.env.j2 b/roles/deepflow_server/templates/deepflow.env.j2 index 943ff42..61b6c62 100644 --- a/roles/deepflow_server/templates/deepflow.env.j2 +++ b/roles/deepflow_server/templates/deepflow.env.j2 @@ -7,13 +7,16 @@ DEEPFLOW_APP_IMAGE={{ deepflow_app_image }} DEEPFLOW_SERVER_GRPC_PORT={{ deepflow_server_grpc_port }} DEEPFLOW_SERVER_HTTP_PORT={{ deepflow_server_http_port }} DEEPFLOW_APP_PORT={{ deepflow_app_port }} +DEEPFLOW_SERVER_CONFIG_FILE={{ deepflow_server_config_file }} DEEPFLOW_CLICKHOUSE_ADDR={{ deepflow_clickhouse_addr }} +DEEPFLOW_CLICKHOUSE_DATABASE={{ deepflow_clickhouse_database }} DEEPFLOW_S3_ENDPOINT={{ deepflow_s3_endpoint }} DEEPFLOW_S3_BUCKET={{ deepflow_s3_bucket }} DEEPFLOW_S3_ACCESS_KEY={{ deepflow_s3_access_key }} DEEPFLOW_S3_SECRET_KEY={{ deepflow_s3_secret_key }} DEEPFLOW_S3_REGION={{ deepflow_s3_region }} +DEEPFLOW_S3_ENABLED={{ deepflow_s3_enabled | ternary('true', 'false') }} DEEPFLOW_MYSQL_ADDR={{ deepflow_mysql_addr }} DEEPFLOW_MYSQL_USER={{ deepflow_mysql_user }} diff --git a/roles/deepflow_server/templates/docker-compose.yml.j2 b/roles/deepflow_server/templates/docker-compose.yml.j2 index 861f1a6..6550a93 100644 --- a/roles/deepflow_server/templates/docker-compose.yml.j2 +++ b/roles/deepflow_server/templates/docker-compose.yml.j2 @@ -18,11 +18,22 @@ services: DEEPFLOW_MYSQL_USER: ${DEEPFLOW_MYSQL_USER} DEEPFLOW_MYSQL_PASSWORD: ${DEEPFLOW_MYSQL_PASSWORD} DEEPFLOW_MYSQL_DATABASE: ${DEEPFLOW_MYSQL_DATABASE} +{% for key, value in (deepflow_server_extra_env | default({})).items() %} + {{ key }}: {{ value | to_json }} +{% endfor %} ports: - '${DEEPFLOW_SERVER_GRPC_PORT}:20035' - '${DEEPFLOW_SERVER_HTTP_PORT}:20417' volumes: - ${DEEPFLOW_DATA}/server:/var/lib/deepflow + - ${DEEPFLOW_SERVER_CONFIG_FILE}:/etc/deepflow/server.yaml:ro + - ${DEEPFLOW_SERVER_CONFIG_FILE}:/etc/server.yaml:ro +{% if deepflow_server_extra_labels | default({}) %} + labels: +{% for key, value in (deepflow_server_extra_labels | default({})).items() %} + {{ key }}: {{ value | to_json }} +{% endfor %} +{% endif %} deepflow-app: container_name: deepflow-app diff --git a/roles/deepflow_server/templates/server.yaml.j2 b/roles/deepflow_server/templates/server.yaml.j2 new file mode 100644 index 0000000..d715be8 --- /dev/null +++ b/roles/deepflow_server/templates/server.yaml.j2 @@ -0,0 +1,27 @@ +spec: + listen-port: {{ deepflow_server_http_port }} + listen-node-port: {{ deepflow_server_grpc_port }} + listen-node-ip: {{ deepflow_server_listen_ip | quote }} +mysql: + host: {{ deepflow_mysql_addr.split(':')[0] | quote }} + port: {{ (deepflow_mysql_addr.split(':') | length > 1) | ternary(deepflow_mysql_addr.split(':')[1], '3306') }} + user: {{ deepflow_mysql_user | quote }} + password: {{ deepflow_mysql_password | quote }} + database: {{ deepflow_mysql_database | quote }} +clickhouse: + host: {{ deepflow_clickhouse_addr.split(':')[0] | quote }} + port: {{ (deepflow_clickhouse_addr.split(':') | length > 1) | ternary(deepflow_clickhouse_addr.split(':')[1], '9000') }} + database: {{ deepflow_clickhouse_database | quote }} + flow-metrics-ttl-hour: {{ deepflow_flow_metrics_ttl_hour }} + metrics-ttl-hour: {{ deepflow_metrics_ttl_hour }} + l4-log-ttl-hour: {{ deepflow_l4_log_ttl_hour }} + l7-log-ttl-hour: {{ deepflow_l7_log_ttl_hour }} + prometheus-ttl-hour: {{ deepflow_prometheus_ttl_hour }} +{% if deepflow_s3_enabled | bool %} +s3: + endpoint: {{ deepflow_s3_endpoint | quote }} + bucket: {{ deepflow_s3_bucket | quote }} + ak: {{ deepflow_s3_access_key | quote }} + sk: {{ deepflow_s3_secret_key | quote }} + region: {{ deepflow_s3_region | quote }} +{% endif %}