From b4d011ca4eec725b8f4fef0559b414190a250151 Mon Sep 17 00:00:00 2001 From: Haitao Pan Date: Wed, 4 Feb 2026 00:41:36 +0800 Subject: [PATCH] feat(obs): template self-node monitoring and local ingest on observability host --- roles/infra/tasks/register.yml | 24 +++++++++++++++++++++++- roles/node_monitor/README.md | 2 +- roles/node_monitor/tasks/main.yml | 6 +++--- scripts/agent-install.sh | 18 ++++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/roles/infra/tasks/register.yml b/roles/infra/tasks/register.yml index c32d0ad..302e3c4 100644 --- a/roles/infra/tasks/register.yml +++ b/roles/infra/tasks/register.yml @@ -9,6 +9,28 @@ loop: '{{ groups["infra"]|default([]) }}' template: src=prometheus/infra.yml.j2 dest=/infra/targets/infra/infra-{{ infra_seq }}.yml owner=victoria group=infra mode=0640 +# Keep infra host self-node metrics registration managed in infra role. +# This avoids dependence on node_monitor completion order for local node target. +- name: render infra self node target to victoria + tags: [ infra_register, register, add_metrics ] + ignore_errors: true + delegate_to: '{{ item }}' + loop: '{{ groups["infra"]|default([]) }}' + copy: + dest: /infra/targets/node/{{ item }}.yml + owner: victoria + group: infra + mode: '0640' + content: | + # {{ item }} + - labels: + ip: "{{ item }}" + ins: "{{ hostvars[item]['nodename'] | default(item) }}" + host: "{{ hostvars[item]['ansible_hostname'] | default(hostvars[item]['nodename'] | default(item)) }}" + cls: "{{ hostvars[item]['node_cluster'] | default('infra') }}" + targets: + - "{{ item }}:{{ hostvars[item]['node_exporter_port'] | default(9100) }}" + #--------------------------------------------------------------# # Register infra logs to vector [add_logs][infra_register] #--------------------------------------------------------------# @@ -138,4 +160,4 @@ gf_pass: "{{ grafana_admin_password|default('pigsty') }}" -... \ No newline at end of file +... diff --git a/roles/node_monitor/README.md b/roles/node_monitor/README.md index 2a3ab03..17d440f 100644 --- a/roles/node_monitor/README.md +++ b/roles/node_monitor/README.md @@ -150,7 +150,7 @@ useful ones (tcpstat for TCP connection stats, processes for process metrics). Creates target files at `/infra/targets/node/.yml`: ```yaml -- labels: { ip: 10.10.10.11, ins: pg-test-1, cls: pg-test } +- labels: { ip: 10.10.10.11, ins: pg-test-1, host: pg-test-host, cls: pg-test } targets: - 10.10.10.11:9100 # node_exporter - 10.10.10.11:9101 # haproxy_exporter diff --git a/roles/node_monitor/tasks/main.yml b/roles/node_monitor/tasks/main.yml index fe2bc70..b90bc7c 100644 --- a/roles/node_monitor/tasks/main.yml +++ b/roles/node_monitor/tasks/main.yml @@ -151,7 +151,7 @@ content: | # {{ inventory_hostname }} # node, haproxy, vector - - labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , cls: {{ node_cluster|default('nodes') }} } + - labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , host: {{ ansible_hostname|default(nodename|default(inventory_hostname)) }} , cls: {{ node_cluster|default('nodes') }} } targets: {% if not node_exporter_enabled|bool and not haproxy_enabled|bool and not vector_enabled|bool %}[]{% endif %} {% if node_exporter_enabled|bool %}- {{ inventory_hostname }}:{{ node_exporter_port }}{% endif %} @@ -161,7 +161,7 @@ {% if vip_enabled|bool and vip_address is defined and vip_address != '' %} # keepalived - - labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , cls: {{ node_cluster|default('nodes') }}, vip: {{ vip_address }} } + - labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , host: {{ ansible_hostname|default(nodename|default(inventory_hostname)) }} , cls: {{ node_cluster|default('nodes') }}, vip: {{ vip_address }} } targets: [ {{ inventory_hostname }}:{{ vip_exporter_port }} ] {% endif %} @@ -204,4 +204,4 @@ - import_tasks: vector.yml tags: vector when: vector_enabled|bool -... \ No newline at end of file +... diff --git a/scripts/agent-install.sh b/scripts/agent-install.sh index 6b72b28..1fb632a 100755 --- a/scripts/agent-install.sh +++ b/scripts/agent-install.sh @@ -15,6 +15,8 @@ ACTION="deploy" ENDPOINT="${DEFAULT_ENDPOINT}" METRICS_ENDPOINT="" LOGS_ENDPOINT="" +METRICS_ENDPOINT_SET=false +LOGS_ENDPOINT_SET=false AUTO_YES=false GREEN='\033[0;32m' @@ -81,18 +83,22 @@ while [[ $# -gt 0 ]]; do ;; --metrics-endpoint) METRICS_ENDPOINT="$2" + METRICS_ENDPOINT_SET=true shift 2 ;; --metrics-endpoint=*) METRICS_ENDPOINT="${1#*=}" + METRICS_ENDPOINT_SET=true shift ;; --logs-endpoint) LOGS_ENDPOINT="$2" + LOGS_ENDPOINT_SET=true shift 2 ;; --logs-endpoint=*) LOGS_ENDPOINT="${1#*=}" + LOGS_ENDPOINT_SET=true shift ;; -y|--yes) @@ -122,6 +128,18 @@ if [[ -z "${LOGS_ENDPOINT}" ]]; then LOGS_ENDPOINT="${base_endpoint}/ingest/logs/insert" fi +# observability server should bypass external HTTPS ingress for local self-monitoring +local_host="$(hostname -f 2>/dev/null || hostname)" +local_short="${local_host%%.*}" +if [[ "${local_host}" == "observability.svc.plus" || "${local_short}" == "observability" ]]; then + if [[ "${METRICS_ENDPOINT_SET}" == "false" ]]; then + METRICS_ENDPOINT="http://127.0.0.1:8428/api/v1/write" + fi + if [[ "${LOGS_ENDPOINT_SET}" == "false" ]]; then + LOGS_ENDPOINT="http://127.0.0.1:9428/insert" + fi +fi + if [[ $EUID -ne 0 ]]; then log_error "This script must be run as root" exit 1