feat(obs): template self-node monitoring and local ingest on observability host

roles/infra/tasks/register.yml

@@ -9,6 +9,28 @@
   loop: '{{ groups["infra"]|default([]) }}'
   template: src=prometheus/infra.yml.j2 dest=/infra/targets/infra/infra-{{ infra_seq }}.yml owner=victoria group=infra mode=0640
 
+# Keep infra host self-node metrics registration managed in infra role.
+# This avoids dependence on node_monitor completion order for local node target.
+- name: render infra self node target to victoria
+  tags: [ infra_register, register, add_metrics ]
+  ignore_errors: true
+  delegate_to: '{{ item }}'
+  loop: '{{ groups["infra"]|default([]) }}'
+  copy:
+    dest: /infra/targets/node/{{ item }}.yml
+    owner: victoria
+    group: infra
+    mode: '0640'
+    content: |
+      # {{ item }}
+      - labels:
+          ip: "{{ item }}"
+          ins: "{{ hostvars[item]['nodename'] | default(item) }}"
+          host: "{{ hostvars[item]['ansible_hostname'] | default(hostvars[item]['nodename'] | default(item)) }}"
+          cls: "{{ hostvars[item]['node_cluster'] | default('infra') }}"
+        targets:
+          - "{{ item }}:{{ hostvars[item]['node_exporter_port'] | default(9100) }}"
+
 #--------------------------------------------------------------#
 # Register infra logs to vector       [add_logs][infra_register]
 #--------------------------------------------------------------#
@@ -138,4 +160,4 @@
     gf_pass: "{{ grafana_admin_password|default('pigsty') }}"
 
 
-...
+...

roles/node_monitor/README.md

@@ -150,7 +150,7 @@ useful ones (tcpstat for TCP connection stats, processes for process metrics).
 Creates target files at `/infra/targets/node/<ip>.yml`:
 
 ```yaml
-- labels: { ip: 10.10.10.11, ins: pg-test-1, cls: pg-test }
+- labels: { ip: 10.10.10.11, ins: pg-test-1, host: pg-test-host, cls: pg-test }
   targets:
     - 10.10.10.11:9100    # node_exporter
     - 10.10.10.11:9101    # haproxy_exporter

roles/node_monitor/tasks/main.yml

@@ -151,7 +151,7 @@
     content: |
       # {{ inventory_hostname }}
       # node, haproxy, vector
-      - labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , cls: {{ node_cluster|default('nodes') }} }
+      - labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , host: {{ ansible_hostname|default(nodename|default(inventory_hostname)) }} , cls: {{ node_cluster|default('nodes') }} }
         targets: {% if not node_exporter_enabled|bool and not haproxy_enabled|bool and not vector_enabled|bool %}[]{% endif %} 
           {% if node_exporter_enabled|bool %}- {{ inventory_hostname }}:{{ node_exporter_port }}{% endif %}
       
@@ -161,7 +161,7 @@
       
       {% if vip_enabled|bool and vip_address is defined and vip_address != '' %}
       # keepalived
-      - labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , cls: {{ node_cluster|default('nodes') }}, vip: {{ vip_address }} }
+      - labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , host: {{ ansible_hostname|default(nodename|default(inventory_hostname)) }} , cls: {{ node_cluster|default('nodes') }}, vip: {{ vip_address }} }
         targets: [ {{ inventory_hostname }}:{{ vip_exporter_port }} ]
       {% endif %}
 
@@ -204,4 +204,4 @@
 - import_tasks: vector.yml
   tags: vector
   when: vector_enabled|bool
-...
+...

scripts/agent-install.sh

@@ -15,6 +15,8 @@ ACTION="deploy"
 ENDPOINT="${DEFAULT_ENDPOINT}"
 METRICS_ENDPOINT=""
 LOGS_ENDPOINT=""
+METRICS_ENDPOINT_SET=false
+LOGS_ENDPOINT_SET=false
 AUTO_YES=false
 
 GREEN='\033[0;32m'
@@ -81,18 +83,22 @@ while [[ $# -gt 0 ]]; do
             ;;
         --metrics-endpoint)
             METRICS_ENDPOINT="$2"
+            METRICS_ENDPOINT_SET=true
             shift 2
             ;;
         --metrics-endpoint=*)
             METRICS_ENDPOINT="${1#*=}"
+            METRICS_ENDPOINT_SET=true
             shift
             ;;
         --logs-endpoint)
             LOGS_ENDPOINT="$2"
+            LOGS_ENDPOINT_SET=true
             shift 2
             ;;
         --logs-endpoint=*)
             LOGS_ENDPOINT="${1#*=}"
+            LOGS_ENDPOINT_SET=true
             shift
             ;;
         -y|--yes)
@@ -122,6 +128,18 @@ if [[ -z "${LOGS_ENDPOINT}" ]]; then
     LOGS_ENDPOINT="${base_endpoint}/ingest/logs/insert"
 fi
 
+# observability server should bypass external HTTPS ingress for local self-monitoring
+local_host="$(hostname -f 2>/dev/null || hostname)"
+local_short="${local_host%%.*}"
+if [[ "${local_host}" == "observability.svc.plus" || "${local_short}" == "observability" ]]; then
+    if [[ "${METRICS_ENDPOINT_SET}" == "false" ]]; then
+        METRICS_ENDPOINT="http://127.0.0.1:8428/api/v1/write"
+    fi
+    if [[ "${LOGS_ENDPOINT_SET}" == "false" ]]; then
+        LOGS_ENDPOINT="http://127.0.0.1:9428/insert"
+    fi
+fi
+
 if [[ $EUID -ne 0 ]]; then
     log_error "This script must be run as root"
     exit 1