---
#==============================================================#
# File      :   etcd.yml
# Ctime     :   2023-02-24
# Mtime     :   2024-09-25
# Desc      :   Record rules for etcd
# Path      :   /infra/rules/etcd.yml
# License   :   Apache-2.0 @ https://pigsty.io/docs/about/license/
# Copyright :   2018-2026  Ruohang Feng / Vonng (rh@vonng.com)
#==============================================================#


groups:
  ################################################################
  #                         Etcd Rules                           #
  ################################################################
  - name: etcd-rules
    rules:

      # etcd client requests per minute
      - record: etcd:ins:rpm
        expr: sum without (type, client_api_version) (increase(etcd_server_client_requests_total{job="etcd"}[1m]))

      # etcd write per minute
      - record: etcd:ins:wpm
        expr: increase(etcd_server_proposals_committed_total{job="etcd"}[1m])

      # etcd db size quota usage
      - record: etcd:ins:quota_usage
        expr: etcd_mvcc_db_total_size_in_bytes{job="etcd"} / etcd_server_quota_backend_bytes{job="etcd"}
      - record: etcd:cls:quota_usage
        expr: max by (job,cls) (etcd_mvcc_db_total_size_in_bytes{job="etcd"} / etcd_server_quota_backend_bytes{job="etcd"})

      # etcd current revision - compact revision
      - record: etcd:ins:age
        expr: etcd_debugging_mvcc_current_revision{job="etcd"} - etcd_debugging_mvcc_compact_revision{job="etcd"}
      - record: etcd:cls:age
        expr: max by (job,cls) (etcd_debugging_mvcc_current_revision{job="etcd"} - etcd_debugging_mvcc_compact_revision{job="etcd"})

      # etcd wal fsync duration (p95 in last 5m)
      - record: etcd:ins:wal_fsync_rt_p95_5m
        expr: histogram_quantile(0.95, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job="etcd"}[5m]))

      # etcd backend commit rt (p95 in last 5m)
      - record: etcd:ins:backend_commit_rt_p95_5m
        expr: histogram_quantile(0.95, rate(etcd_disk_backend_commit_duration_seconds_bucket{job="etcd"}[5m]))

      # etcd network peer rt (p95 in last 5m)
      - record: etcd:ins:network_peer_rt_p95_5m
        expr: histogram_quantile(0.95, rate(etcd_network_peer_round_trip_time_seconds_bucket{job="etcd"}[5m]))


  ################################################################
  #                         Etcd Alert                           #
  ################################################################
  - name: etcd-alert
    rules:

      #==============================================================#
      #                         Aliveness                            #
      #==============================================================#
      # etcd server instance down
      - alert: EtcdServerDown
        expr: etcd_up < 1
        for: 1m
        labels: { level: 0, severity: CRIT, category: etcd }
        annotations:
          summary: "CRIT EtcdServerDown {{ $labels.ins }}@{{ $labels.instance }}"
          description: |
            etcd_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
            /ui/d/etcd-overview

      #==============================================================#
      #                         Error                                #
      #==============================================================#
      # Etcd no Leader triggers a P0 alert immediately
      # if dcs_failsafe mode is not enabled, this may lead to global outage
      - alert: EtcdNoLeader
        expr: min(etcd_server_has_leader) by (cls) < 1
        for: 15s
        labels: { level: 0, severity: CRIT, category: etcd }
        annotations:
          summary: "CRIT EtcdNoLeader: {{ $labels.cls }} {{ $value }}"
          description: |
            etcd_server_has_leader[cls={{ $labels.cls }}] = {{ $value }} < 1
            /ui/d/etcd-overview?from=now-5m&to=now&var-cls={{$labels.cls}}

      #==============================================================#
      #                        Saturation                            #
      #==============================================================#
      - alert: EtcdQuotaFull
        expr: etcd:cls:quota_usage > 0.90
        for: 1m
        labels: { level: 1, severity: WARN, category: etcd }
        annotations:
          summary: "WARN EtcdQuotaFull: {{ $labels.cls }}"
          description: |
            etcd:cls:quota_usage[cls={{ $labels.cls }}] = {{ $value | printf "%.3f" }} > 90%

      #==============================================================#
      #                         Latency                              #
      #==============================================================#
      # etcd network peer rt p95 > 200ms for 1m
      - alert: EtcdNetworkPeerRTSlow
        expr: etcd:ins:network_peer_rt_p95_5m > 0.200
        for: 1m
        labels: { level: 2, severity: INFO, category: etcd }
        annotations:
          summary: "INFO EtcdNetworkPeerRTSlow: {{ $labels.cls }} {{ $labels.ins }}"
          description: |
            etcd:ins:network_peer_rt_p95_5m[cls={{ $labels.cls }}, ins={{ $labels.ins }}] = {{ $value }} > 200ms
            /ui/d/etcd-instance?from=now-10m&to=now&var-cls={{ $labels.cls }}

      # Etcd wal fsync rt p95 > 50ms
      - alert: EtcdWalFsyncSlow
        expr: etcd:ins:wal_fsync_rt_p95_5m > 0.050
        for: 1m
        labels: { level: 2, severity: INFO, category: etcd }
        annotations:
          summary: "INFO EtcdWalFsyncSlow: {{ $labels.cls }} {{ $labels.ins }}"
          description: |
            etcd:ins:wal_fsync_rt_p95_5m[cls={{ $labels.cls }}, ins={{ $labels.ins }}] = {{ $value }} > 50ms
            /ui/d/etcd-instance?from=now-10m&to=now&var-cls={{ $labels.cls }}

...
