---
#==============================================================#
# File      :   redis.yml
# Ctime     :   2020-04-22
# Mtime     :   2024-08-30
# Desc      :   Record rules for redis
# Path      :   /infra/rules/redis.yml
# License   :   Apache-2.0 @ https://pigsty.io/docs/about/license/
# Copyright :   2018-2026  Ruohang Feng / Vonng (rh@vonng.com)
#==============================================================#

################################################################
#                     Alert Rule Format                        #
################################################################
# Level: three alert levels
#   0 : CRIT System failure needs immediate intervene   call  (e.g down) 1m
#   1 : WARN Anomalies could lead to failure soon       sms   (e.g high) 1m
#   2 : INFO Warning events that need attention         mail             5m

# - alert: RedisDown        <------- CamelCase Alert Name
#   expr: redis_up < 1      <------- Expression
#   for: 1m                 <------- Duration Threshold
#        ^------- (omit) : Trigger immediately
#   labels:
#     level: 0              <------- numeric expression of severity 0,1,2
#     severity: CRIT        <------- alert severity: fatal,error,event
#     category: redis       <------- category: infra, node, pgsql, redis, ...
#   annotations:            <------- short & detailed information about context
#     summary: "FATAL RedisDown {{ $labels.type }}@{{ $labels.instance }}"
#     description: |
#       redis_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
################################################################


groups:
  ################################################################
  #                         Redis Rules                          #
  ################################################################
  - name: redis-rules
    rules:

      #==============================================================#
      #                         Identity                             #
      #==============================================================#
      # redis:ins holds id(ins,instance) map to redis instance
      - record: redis:ins
        expr: |
          max by (id, ins, instance, cls, job) (
            label_replace(redis_up, "id", "$1", "ins", "(.+)") OR
            label_replace(redis_up, "id", "$1", "instance", "(.+)")
          )

      #==============================================================#
      #                           Node                               #
      #==============================================================#
      # add new `ip` label to redis_up
      - record: redis:node:ip
        expr: label_replace(redis_up, "ip", "$1", "instance", "(.*)\\:\\d+")

      # node memory total bytes
      - record: redis:node:mem_total
        expr: max by (job,cls,ip) (label_replace(redis_total_system_memory_bytes, "ip", "$1", "instance", "(.*)\\:\\d+"))

      # config max memory for all instances on node
      - record: redis:node:mem_alloc
        expr: sum by (job,cls,ip) (label_replace(redis_config_maxmemory, "ip", "$1", "instance", "(.*)\\:\\d+"))

      - record: redis:node:mem_used
        expr: sum by (job,cls,ip) (label_replace(redis_memory_used_bytes, "ip", "$1", "instance", "(.*)\\:\\d+"))

      # config max memory for all instances on node
      - record: redis:node:qps
        expr: sum by (job,cls,ip) (label_replace(redis:ins:qps, "ip", "$1", "instance", "(.*)\\:\\d+"))


      #==============================================================#
      #                           CPU                                #
      #==============================================================#
      # cpu usage
      - record: redis:ins:cpu_sys
        expr: rate(redis_cpu_sys_seconds_total[1m])
      - record: redis:ins:cpu_user
        expr: rate(redis_cpu_user_seconds_total[1m])
      - record: redis:ins:cpu_usage
        expr: rate(redis_cpu_sys_seconds_total[1m]) + rate(redis_cpu_user_seconds_total[1m])

      # cpu usage (cluster)
      - record: redis:cls:cpu_sys
        expr: avg by (job, cls) (redis:ins:cpu_sys)
      - record: redis:cls:cpu_user
        expr: avg by (job, cls) (redis:ins:cpu_user)
      - record: redis:cls:cpu_usage
        expr: avg by (job, cls) (redis:ins:cpu_usage)

      # cpu usage (overall)
      - record: redis:env:cpu_usage
        expr: avg by (job)(redis:ins:cpu_usage)

      # cpu usage (child)
      - record: redis:ins:cpu_sys_child
        expr: rate(redis_cpu_sys_children_seconds_total[1m])
      - record: redis:ins:cpu_user_child
        expr: rate(redis_cpu_user_children_seconds_total[1m])
      - record: redis:ins:cpu_usage_child
        expr: rate(redis_cpu_sys_children_seconds_total[1m]) + rate(redis_cpu_user_children_seconds_total[1m])

      # cpu usage (child, cluster)
      - record: redis:cls:cpu_sys_child
        expr: avg by (job, cls) (redis:ins:cpu_sys_child)
      - record: redis:cls:cpu_user_child
        expr: avg by (job, cls) (redis:ins:cpu_user_child)
      - record: redis:cls:cpu_usage_child
        expr: avg by (job, cls) (redis:ins:cpu_usage_child)

      # cpu usage (child, overall)
      - record: redis:env:cpu_usage_child
        expr: avg by (job) (redis:ins:cpu_usage_child)

      #==============================================================#
      #                          Memory                              #
      #==============================================================#
      # mem usage
      - record: redis:ins:mem_usage
        expr: redis_memory_used_bytes / redis_memory_max_bytes

      # mem usage (cluster)
      - record: redis:cls:mem_used
        expr: sum by (job, cls) (redis_memory_used_bytes)
      - record: redis:cls:mem_max
        expr: sum by (job, cls) (redis_memory_max_bytes)
      - record: redis:cls:mem_usage
        expr: redis:cls:mem_used / redis:cls:mem_max
      - record: redis:cls:mem_usage_max
        expr: max by (job, cls) (redis:ins:mem_usage)
      
      # mem usage (overall)
      - record: redis:env:mem_usage
        expr: sum by (job)(redis:cls:mem_used) / sum by (job)(redis:cls:mem_max)


      #==============================================================#
      #                           Network                            #
      #==============================================================#
      # receive bandwidth (in)
      - record: redis:ins:rx
        expr: rate(redis_net_input_bytes_total[1m])
      - record: redis:cls:rx
        expr: sum by (job, cls) (redis:ins:rx)
      - record: redis:env:rx
        expr: sum by (job) (redis:cls:rx)

      # transmit bandwidth (out)
      - record: redis:ins:tx
        expr: rate(redis_net_output_bytes_total[1m])
      - record: redis:cls:tx
        expr: sum by (job, cls) (redis:ins:tx)
      - record: redis:env:tx
        expr: sum by (job) (redis:cls:tx)

      # network traffic (i+o)
      - record: redis:ins:net_traffic
        expr: redis:ins:rx + redis:ins:tx
      - record: redis:cls:net_traffic
        expr: redis:cls:rx + redis:cls:tx
      - record: redis:env:net_traffic
        expr: redis:env:rx + redis:env:tx

      #==============================================================#
      #                         Activity                             #
      #==============================================================#
      # qps realtime
      - record: redis:ins:qps_realtime
        expr: irate(redis_commands_processed_total[1m])
      - record: redis:cls:qps_realtime
        expr: sum by (job, cls) (redis:ins:qps_realtime)
      - record: redis:env:qps_realtime
        expr: sum by (job) (redis:cls:qps_realtime)
      # qps rate1m
      - record: redis:ins:qps
        expr: rate(redis_commands_processed_total[1m])
      - record: redis:cls:qps
        expr: sum by (job, cls) (redis:ins:qps)
      - record: redis:env:qps
        expr: sum by (job) (redis:cls:qps)
      # qps µ
      - record: redis:ins:qps_mu
        expr: avg_over_time(redis:ins:qps[30m])
      - record: redis:cls:qps_mu
        expr: avg_over_time(redis:cls:qps[30m])
      - record: redis:env:qps_mu
        expr: avg_over_time(redis:env:qps[30m])
      # qps σ
      - record: redis:ins:qps_sigma
        expr: stddev_over_time(redis:ins:qps[30m])
      - record: redis:cls:qps_sigma
        expr: stddev_over_time(redis:cls:qps[30m])
      - record: redis:env:qps_sigma
        expr: stddev_over_time(redis:env:qps[30m])

      #==============================================================#
      #                           Database                           #
      #==============================================================#
      # key expire
      - record: redis:ins:key_expire
        expr: rate(redis_expired_keys_total[1m])
      - record: redis:cls:key_expire
        expr: sum by (job, cls) (redis:ins:key_expire)
      - record: redis:env:key_expire
        expr: sum by (job) (redis:cls:key_expire)
      # key evict
      - record: redis:ins:key_evict
        expr: rate(redis_evicted_keys_total[1m])
      - record: redis:cls:key_evict
        expr: sum by (job, cls) (redis:ins:key_evict)
      - record: redis:env:key_evict
        expr: sum by (job) (redis:cls:key_evict)
      # key hit
      - record: redis:ins:key_hit
        expr: rate(redis_keyspace_hits_total[1m])
      - record: redis:cls:key_hit
        expr: sum by (job, cls) (redis:ins:key_hit)
      - record: redis:env:key_hit
        expr: sum by (job) (redis:cls:key_hit)
      # key miss
      - record: redis:ins:key_miss
        expr: rate(redis_keyspace_misses_total[1m])
      - record: redis:cls:key_miss
        expr: sum by (job, cls) (redis:ins:key_miss)
      - record: redis:env:key_miss
        expr: sum by (job) (redis:cls:key_miss)
      # key hit rate
      - record: redis:ins:key_hit_rate
        expr: redis:ins:key_hit / (redis:ins:key_hit + redis:ins:key_miss)
      - record: redis:cls:key_hit_rate
        expr: redis:cls:key_hit / (redis:cls:key_hit + redis:cls:key_miss)
      - record: redis:env:key_hit_rate
        expr: redis:env:key_hit / (redis:env:key_hit + redis:env:key_miss)

      #==============================================================#
      #                         Queries                             #
      #==============================================================#
      # cmd time spent
      - record: redis:ins:cmd_time
        expr: rate(redis_commands_duration_seconds_total[1m])
      - record: redis:cls:cmd_time
        expr: sum by (job,cls,cmd) (redis:ins:cmd_time)
      - record: redis:env:cmd_time
        expr: sum by (job,cmd) (redis:ins:cmd_time)
      # cmd qps
      - record: redis:ins:cmd_qps
        expr: rate(redis_commands_total[1m])
      - record: redis:cls:cmd_qps
        expr: sum by (job,cls,cmd) (redis:ins:cmd_qps)
      - record: redis:env:cmd_qps
        expr: sum by (job,cmd) (redis:cls:cmd_qps)
      # command rt
      - record: redis:ins:cmd_rt
        expr: redis:ins:cmd_time / redis:ins:cmd_qps
      - record: redis:cls:cmd_rt
        expr: redis:cls:cmd_time / redis:cls:cmd_qps
      - record: redis:env:cmd_rt
        expr: redis:env:cmd_time / redis:env:cmd_qps
      # command rt agg
      - record: redis:ins:rt
        expr: sum without (cmd) (redis:ins:cmd_time) / sum without (cmd) (redis:ins:cmd_qps)
      - record: redis:cls:rt
        expr: sum without (cmd) (redis:cls:cmd_time) / sum without (cmd) (redis:cls:cmd_qps)
      - record: redis:env:rt
        expr: sum without (cmd) (redis:env:cmd_time) / sum without (cmd) (redis:env:cmd_qps)
      # rt µ
      - record: redis:ins:rt_mu
        expr: avg_over_time(redis:ins:rt[30m])
      - record: redis:cls:rt_mu
        expr: avg_over_time(redis:cls:rt[30m])
      - record: redis:env:rt_mu
        expr: avg_over_time(redis:env:rt[30m])
      # rt σ
      - record: redis:ins:rt_sigma
        expr: stddev_over_time(redis:ins:rt[30m])
      - record: redis:cls:rt_sigma
        expr: stddev_over_time(redis:cls:rt[30m])
      - record: redis:env:rt_sigma
        expr: stddev_over_time(redis:env:rt[30m])


      #==============================================================#
      #                           Session                            #
      #==============================================================#
      # clients
      - record: redis:ins:clients
        expr: redis_connected_clients
      - record: redis:cls:clients
        expr: sum by (job, cls) (redis:ins:clients)
      - record: redis:env:clients
        expr: sum by (job)(redis:cls:clients)
      # blocked clients
      - record: redis:ins:blocked_clients
        expr: rate(redis_blocked_clients[1m])
      - record: redis:cls:blocked_clients
        expr: sum by (job, cls) (redis:ins:blocked_clients)
      - record: redis:env:blocked_clients
        expr: sum by (job)(redis:cls:blocked_clients)
      # conn rate
      - record: redis:ins:conn_rate
        expr: rate(redis_connections_received_total[1m])
      - record: redis:cls:conn_rate
        expr: sum by (job, cls) (redis:ins:conn_rate)
      - record: redis:env:conn_rate
        expr: sum by (job)(redis:cls:conn_rate)
      # conn reject rate
      - record: redis:ins:conn_reject
        expr: rate(redis_rejected_connections_total[1m])
      - record: redis:cls:conn_reject
        expr: sum by (job, cls) (redis:ins:conn_reject)
      - record: redis:env:conn_reject
        expr: sum by (job)(redis:cls:conn_reject)

      #==============================================================#
      #                         Replication                          #
      #==============================================================#
      - record: redis:ins:lsn_rate
        expr: rate(redis_master_repl_offset[1m])


      #==============================================================#
      #                           Persist                            #
      #==============================================================#
      - record: redis:cls:fork_time
        expr: max by (job, cls) (redis_latest_fork_seconds)
      - record: redis:cls:aof_rewrite_time
        expr: max by (job, cls) (redis_aof_last_rewrite_duration_sec)
      - record: redis:cls:size
        expr: count by (job, cls) (redis_up)


  ################################################################
  #                         Redis Alert                          #
  ################################################################
  - name: redis-alert
    rules:

      #==============================================================#
      #                         Error                                #
      #==============================================================#
      # redis down triggers a P0 alert
      - alert: RedisDown
        expr: redis_up < 1
        for: 1m
        labels: { level: 0, severity: CRIT, category: redis }
        annotations:
          summary: "CRIT RedisDown: {{ $labels.ins }} {{ $labels.instance }} {{ $value }}"
          description: |
            redis_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} == 0
            /ui/d/redis-instance?from=now-5m&to=now&var-ins={{$labels.ins}}

      # redis reject connection in last 5m
      - alert: RedisRejectConn
        expr: redis:ins:conn_reject > 0
        labels: { level: 0, severity: CRIT, category: redis }
        annotations:
          summary: "CRIT RedisRejectConn: {{ $labels.ins }} {{ $labels.instance }} {{ $value }}"
          description: |
            redis:ins:conn_reject[cls={{ $labels.cls }}, ins={{ $labels.ins }}][5m] = {{ $value }} > 0
            /ui/d/redis-instance?from=now-10m&to=now&viewPanel=88&fullscreen&var-ins={{ $labels.ins }}



      #==============================================================#
      #                         Latency                              #
      #==============================================================#
      # redis avg query response time > 160 µs
      - alert: RedisRTHigh
        expr: redis:ins:rt > 0.00016
        for: 1m
        labels: { level: 1, severity: WARN, category: redis }
        annotations:
          summary: "WARN RedisRTHigh: {{ $labels.cls }} {{ $labels.ins }}"
          description: |
            pg:ins:query_rt[cls={{ $labels.cls }}, ins={{ $labels.ins }}] = {{ $value }} > 160µs
            /ui/d/redis-instance?from=now-10m&to=now&viewPanel=97&fullscreen&var-ins={{ $labels.ins }}



      #==============================================================#
      #                        Saturation                            #
      #==============================================================#
      # redis cpu usage more than 70% for 1m
      - alert: RedisCPUHigh
        expr: redis:ins:cpu_usage > 0.70
        for: 1m
        labels: { level: 1, severity: WARN, category: redis }
        annotations:
          summary: "WARN RedisCPUHigh: {{ $labels.cls }} {{ $labels.ins }}"
          description: |
            redis:ins:cpu_all[cls={{ $labels.cls }}, ins={{ $labels.ins }}] = {{ $value }} > 60%
            /ui/d/redis-instance?from=now-10m&to=now&viewPanel=43&fullscreen&var-ins={{ $labels.ins }}

      # redis mem usage more than 70% for 1m
      - alert: RedisMemHigh
        expr: redis:ins:mem_usage > 0.70
        for: 1m
        labels: { level: 1, severity: WARN, category: redis }
        annotations:
          summary: "WARN RedisMemHigh: {{ $labels.cls }} {{ $labels.ins }}"
          description: |
            redis:ins:mem_usage[cls={{ $labels.cls }}, ins={{ $labels.ins }}] = {{ $value }} > 80%
            /ui/d/redis-instance?from=now-10m&to=now&viewPanel=7&fullscreen&var-ins={{ $labels.ins }}

      #==============================================================#
      #                         Traffic                              #
      #==============================================================#
      # redis qps more than 32000 for 5m
      - alert: RedisQPSHigh
        expr: redis:ins:qps > 32000
        for: 5m
        labels: { level: 2, severity: INFO, category: redis }
        annotations:
          summary: "INFO RedisQPSHigh: {{ $labels.cls }} {{ $labels.ins }}"
          description: |
            redis:ins:qps[cls={{ $labels.cls }}, ins={{ $labels.ins }}] = {{ $value }} > 16000
            /ui/d/redis-instance?from=now-10m&to=now&viewPanel=96&fullscreen&var-ins={{ $labels.ins }}
...
