observability.svc.plus/roles/node_monitor/tasks/main.yml
2026-02-01 20:53:55 +08:00

207 lines
8.3 KiB
YAML

---
#--------------------------------------------------------------#
# Register HAProxy to Nginx [haproxy_register]
#--------------------------------------------------------------#
# nginx are idempotent on multiple meta nodes
- name: register haproxy instance to nginx
tags: [ haproxy_register, register_nginx, register, add_proxy ]
when: haproxy_enabled|bool
become: true
block:
- name: create nginx config dir for haproxy
run_once: true
delegate_to: '{{ item }}'
loop: '{{ groups["infra"]|default([]) }}'
file: path=/etc/nginx/conf.d/haproxy state=directory owner=root
# /etc/nginx/conf.d/haproxy/upstream-{{ nodename }}.conf
- name: register haproxy upstream to nginx
delegate_to: '{{ item }}'
loop: '{{ groups["infra"]|default([]) }}'
copy:
dest: /etc/nginx/conf.d/haproxy/upstream-{{ nodename }}.conf
content: |
upstream haproxy-{{ nodename }} {
server {{ inventory_hostname }}:{{ haproxy_exporter_port }} max_fails=0;
}
# /etc/nginx/conf.d/haproxy/location-{{ nodename }}.conf
- name: register haproxy url location to nginx
delegate_to: '{{ item }}'
loop: '{{ groups["infra"]|default([]) }}'
copy:
dest: /etc/nginx/conf.d/haproxy/location-{{ nodename }}.conf
content: |
location ^~/haproxy/{{ nodename }}/ {
proxy_pass http://haproxy-{{ nodename }};
proxy_connect_timeout 1;
}
# reload meta node nginx (maybe not appropriate here)
- name: reload nginx to finish haproxy register
delegate_to: '{{ item }}'
run_once: true
loop: '{{ groups["infra"]|default([]) }}'
systemd: name=nginx state=reloaded enabled=yes daemon_reload=yes
#--------------------------------------------------------------#
# Register Instance DNS Name [vip_dns]
#--------------------------------------------------------------#
# render to temp file first, then atomic mv to avoid dnsmasq inotify race
# use {{ node_cluster }}.vip as filename to avoid conflict with pg_cluster dns
- name: render node vip dns name
tags: [ node_vip, vip_dns, add_dns ]
when: vip_enabled|bool
delegate_to: '{{ item }}'
loop: "{{ groups['infra'] | default([]) }}"
ignore_errors: true
copy:
dest: /infra/hosts/.{{ node_cluster }}.vip.tmp
mode: 0644
owner: root
group: root
content: "{{ vip_address }} {{ node_cluster }}{{ vip_dns_suffix|default('') }}"
- name: activate node vip dns name
tags: [ node_vip, vip_dns, add_dns ]
when: vip_enabled|bool
delegate_to: '{{ item }}'
loop: "{{ groups['infra'] | default([]) }}"
ignore_errors: true
shell: chcon -t dnsmasq_etc_t /infra/hosts/.{{ node_cluster }}.vip.tmp 2>/dev/null; mv /infra/hosts/.{{ node_cluster }}.vip.tmp /infra/hosts/{{ node_cluster }}.vip || true
args: { executable: /bin/bash }
#--------------------------------------------------------------#
# Config node_exporter [node_exporter_config]
#--------------------------------------------------------------#
- name: config node_exporter
tags: [ node_exporter, node_exporter_config ]
block:
- name: config node_exporter systemd unit
copy: src=node_exporter.svc dest={{ systemd_dir }}/node_exporter.service
- name: config default node_exporter options
copy:
dest: /etc/default/node_exporter
content: |
NODE_EXPORTER_OPTS="--web.listen-address=':{{ node_exporter_port }}' --web.telemetry-path='{{ exporter_metrics_path }}' {{ node_exporter_options }}"
#--------------------------------------------------------------#
# Launch node_exporter [node_exporter_launch]
#--------------------------------------------------------------#
- name: launch node_exporter
tags: [ node_exporter, node_exporter_launch ]
when: node_exporter_enabled|bool
block:
- name: launch node_exporter systemd service
systemd: name=node_exporter state=restarted enabled=yes daemon_reload=yes
- name: wait for node_exporter service online
wait_for: host=127.0.0.1 port={{ node_exporter_port }} state=started timeout=10
#--------------------------------------------------------------#
# Config keepalived_exporter [vip_exporter_config]
#--------------------------------------------------------------#
- name: config keepalived_exporter
tags: [ node_vip, vip_exporter, vip_exporter_config ]
when: vip_enabled|bool
block:
- name: config keepalived_exporter systemd unit
copy: src=keepalived_exporter.svc dest={{ systemd_dir }}/keepalived_exporter.service
- name: config default keepalived_exporter options
copy:
dest: /etc/default/keepalived_exporter
content: |
KEEPALIVED_EXPORTER_OPTS="--web.listen-address=':{{ vip_exporter_port }}' --web.telemetry-path='{{ exporter_metrics_path }}'"
#--------------------------------------------------------------#
# Launch keepalived_exporter [vip_exporter_launch]
#--------------------------------------------------------------#
- name: launch keepalived_exporter
tags: [ node_vip, vip_exporter, vip_exporter_launch ]
when: vip_enabled|bool
block:
- name: launch keepalived_exporter systemd service
systemd: name=keepalived_exporter state=restarted enabled=yes daemon_reload=yes
- name: wait for keepalived_exporter service online
wait_for: host=127.0.0.1 port={{ vip_exporter_port }} state=started timeout=10
#--------------------------------------------------------------#
# Register Node [node_register]
#--------------------------------------------------------------#
# /infra/targets/node/{{ ip }}.yml
- name: register node as victoria target
tags: [ node_vip, node_register, register, add_metrics ]
ignore_errors: true
delegate_to: '{{ item }}'
loop: '{{ groups["infra"]|default([]) }}'
copy:
dest: "/infra/targets/node/{{ inventory_hostname }}.yml"
owner: victoria
group: infra
mode: '0640'
content: |
# {{ inventory_hostname }}
# node, haproxy, vector
- labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , cls: {{ node_cluster|default('nodes') }} }
targets: {% if not node_exporter_enabled|bool and not haproxy_enabled|bool and not vector_enabled|bool %}[]{% endif %}
{% if node_exporter_enabled|bool %}- {{ inventory_hostname }}:{{ node_exporter_port }}{% endif %}
{% if haproxy_enabled|bool %}- {{ inventory_hostname }}:{{ haproxy_exporter_port }}{% endif %}
{% if vector_enabled|bool %}- {{ inventory_hostname }}:{{ vector_port }}{% endif %}
{% if vip_enabled|bool and vip_address is defined and vip_address != '' %}
# keepalived
- labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , cls: {{ node_cluster|default('nodes') }}, vip: {{ vip_address }} }
targets: [ {{ inventory_hostname }}:{{ vip_exporter_port }} ]
{% endif %}
- name: register node as ping target
tags: [ node_register, register, add_metrics ]
ignore_errors: true
delegate_to: '{{ item }}'
loop: '{{ groups["infra"]|default([]) }}'
copy:
dest: "/infra/targets/ping/{{ inventory_hostname }}.yml"
owner: victoria
group: infra
mode: '0640'
content: |
# {{ inventory_hostname }}
- labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , cls: {{ node_cluster|default('nodes') }} }
targets: [ {{ inventory_hostname }} ]
- name: register node vip as ping target
tags: [ node_vip, node_register, register, add_metrics ]
when: vip_enabled|bool and vip_address is defined and vip_address != ''
ignore_errors: true
delegate_to: '{{ item }}'
loop: '{{ groups["infra"]|default([]) }}'
copy:
dest: "/infra/targets/ping/{{ vip_address }}---{{ inventory_hostname }}.yml"
owner: victoria
group: infra
mode: '0640'
content: |
# {{ vip_address }}@{{ inventory_hostname }}
- labels: { ip: {{ inventory_hostname }} , ins: {{ nodename }} , cls: {{ node_cluster|default('nodes') }}, vip: {{ vip_address }} , job: node-vip }
targets: [ {{ vip_address }} ]
#--------------------------------------------------------------#
# Vector [vector]
#--------------------------------------------------------------#
- import_tasks: vector.yml
tags: vector
when: vector_enabled|bool
...