---
#==============================================================#
# File      :   infra.yml
# Ctime     :   2020-04-22
# Mtime     :   2021-01-22
# Desc      :   Infrastructure Rules
# Path      :   /infra/rules/infra.yml
# License   :   Apache-2.0 @ https://pigsty.io/docs/about/license/
# Copyright :   2018-2026  Ruohang Feng / Vonng (rh@vonng.com)
#==============================================================#

################################################################
#                     Alert Rule Format                        #
################################################################
# Level: three alert levels
#   0 : CRIT  System failure needs immediate intervene   call  (e.g down) 1m
#   1 : WARN  Anomalies could lead to failure soon       sms   (e.g high) 1m
#   2 : INFO  Warning events that need attention         mail             5m

# - alert: InfraDown        <------- CamelCase Alert Name
#   expr: infra_up < 1      <------- Expression
#   for: 1m                 <------- Duration Threshold
#        ^------- (omit) : Trigger immediately
#   labels:
#     level: 0              <------- numeric expression of severity 0,1,2
#     severity: CRIT        <------- alert severity: fatal,error,event
#     category: infra       <------- category: infra, node, pgsql, redis, ...
#   annotations:            <------- short & detailed information about context
#     summary: "FATAL InfraDown {{ $labels.type }}@{{ $labels.instance }}"
#     description: |
#       infra_up[instance={{ $labels.instance }}] = {{ $value  | printf "%.2f" }} < 1
################################################################


groups:

  ################################################################
  #                Infrastructure Record Rules                   #
  ################################################################
  # prometheus & grafana & alertmanager & nginx & other infra services

  - name: infra-rules
    rules:

      #==============================================================#
      #                      Generic Aliveness                       #
      #==============================================================#
      # infra job monitoring target aliveness
      - record: infra_up
        expr: up{job="infra"}

      # pgsql job related exporter aliveness
      - record: agent_up
        expr: up{job!~"infra|node|etcd|minio|pgsql|redis|mongo|mysql"}


  ################################################################
  #                Infrastructure Alert Rules                    #
  ################################################################
  - name: infra-alert
    rules:


      #==============================================================#
      #                       Infra Aliveness                        #
      #==============================================================#
      # infra components (prometheus,grafana) down for 1m triggers a P1 alert
      - alert: InfraDown
        expr: infra_up < 1
        for: 1m
        labels: { level: 0, severity: CRIT, category: infra }
        annotations:
          summary: "CRIT InfraDown {{ $labels.type }}@{{ $labels.instance }}"
          description: |
            infra_up[type={{ $labels.type }}, instance={{ $labels.instance }}] = {{ $value  | printf "%.2f" }} < 1

      #==============================================================#
      #                       Agent Aliveness                        #
      #==============================================================#

      # agent aliveness are determined directly by exporter aliveness
      # including: node_exporter, pg_exporter, pgbouncer_exporter, haproxy_exporter
      - alert: AgentDown
        expr: agent_up < 1
        for: 1m
        labels: { level: 0, severity: CRIT, category: infra }
        annotations:
          summary: 'CRIT AgentDown {{ $labels.ins }}@{{ $labels.instance }}'
          description: |
            agent_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value  | printf "%.2f" }} < 1

...