{
  "annotations":{
    "list":[
      {"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},
      {"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(vm_app_version{ins=~\"$ins\"}) by(short_version) unless (sum(vm_app_version{ins=~\"$ins\"} offset $__interval) by(short_version))","hide":true,"iconColor":"dark-blue","name":"version","textFormat":"{{short_version}}","titleFormat":"Version change"},
      {"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(changes(vm_app_start_timestamp{ins=~\"$ins\"}[$__interval])) by(job, instance)","hide":false,"iconColor":"dark-yellow","name":"restarts","textFormat":"{{job}}:{{instance}} restarted"}
    ]
  },
  "author":"Ruohang Feng (rh@vonng.com)",
  "description":"Overview for VictoriaMetrics vmalert v1.117.0 or higher",
  "editable":true,
  "fiscalYearStartMonth":0,
  "graphTooltip":1,
  "id":null,
  "license":"https://pigsty.io/docs/about/license/",
  "links":[{"asDropdown":true,"icon":"external link","includeVars":false,"keepTime":true,"tags":["Pigsty","INFRA"],"targetBlank":false,"title":"INFRA","tooltip":"","type":"dashboards","url":""}],
  "panels":[
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":200,"panels":[],"title":"Overview","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]}},"overrides":[{"matcher":{"id":"byName","options":"IP"},"properties":[{"id":"links","value":[{"title":"Node Instance : $ip","url":"/d/node-instance?var-id=${ip}&${__url_time_range}"}]}]}]},
      "gridPos":{"h":6,"w":5,"x":0,"y":1},
      "id":201,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"/.*/","values":true},"showPercentChange":false,"text":{"titleSize":16,"valueSize":16},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"vm_app_version{ins=\"$ins\"}","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],
      "transformations":[
        {
          "id":"organize",
          "options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"cls":true,"job":true,"short_version":false,"version":true},"includeByName":{},"indexByName":{"Time":5,"Value":9,"__name__":6,"cls":3,"ins":0,"instance":1,"ip":2,"job":7,"short_version":4,"version":8},"renameByName":{"Value":"","cls":"Cluster","ins":"Instance","instance":"Address","ip":"IP","job":"","short_version":"Version"}}
        }
      ],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"All VMAlert instances",
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"custom":{"align":"center","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#e3e3e3e0","value":0}]}},
        "overrides":[
          { "matcher":{"id":"byName","options":"Instance"}   ,"properties":[{"id":"links","value":[{"title":"VMAlert Instance ${__data.fields.Instance}","url":"/d/vmalert-instance?var-instance=${__data.fields.Instance}&${__url_time_range}"}]},{"id":"custom.width"}]                                                                                                                                          },
          { "matcher":{"id":"byName","options":"IP"}         ,"properties":[{"id":"custom.width","value":110},{"id":"links","value":[{"title":"Node Instance : ${__data.fields.IP}","url":"/d/node-instance?var-id=${__data.fields.IP}&${__url_time_range}"}]}]                                                                                                                                                    },
          { "matcher":{"id":"byName","options":"Up"}         ,"properties":[{"id":"custom.width","value":50},{"id":"mappings","value":[{"options":{"0":{"color":"#cc4637d9","index":0,"text":"Dead"},"1":{"color":"#346f36cc","index":1,"text":"Alive"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"type":"color-background"}}]                                                                        },
          { "matcher":{"id":"byName","options":"Uptime"}     ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"dtdurations"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"red","value":0},{"color":"#fcdb72","value":300},{"color":"#346f36cc","value":3600}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                                                 },
          { "matcher":{"id":"byName","options":"Alert Rules"},"properties":[{"id":"custom.width","value":90},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":10},{"color":"#fcdb72","value":100},{"color":"#f79f64","value":1000}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                    },
          { "matcher":{"id":"byName","options":"Firing"}     ,"properties":[{"id":"custom.width","value":60},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":10},{"color":"red","value":50}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                            },
          { "matcher":{"id":"byName","options":"Pending"}    ,"properties":[{"id":"custom.width","value":70},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":10}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                                                       },
          { "matcher":{"id":"byName","options":"Memory"}     ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"bytes"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":134217728},{"color":"#fcdb72","value":536870912},{"color":"#f79f64","value":1073741824}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}] }
        ]
      },
      "gridPos":{"h":6,"w":15,"x":5,"y":1},
      "id":202,
      "options":{"cellHeight":"sm","showHeader":true,"sortBy":[{"desc":false,"displayName":"Instance"}]},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min by (ins,ip) (up{cls=\"vmalert\"})"                                         ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"A" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min by (ins) (vm_app_uptime_seconds{cls=\"vmalert\"})"                         ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"B" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (vmalert_alerting_rules_last_evaluation_samples{cls=\"vmalert\"})","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"C" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (vmalert_alerts_firing{cls=\"vmalert\"})"                         ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"D" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (vmalert_alerts_pending{cls=\"vmalert\"})"                        ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"E" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (process_resident_memory_bytes{cls=\"vmalert\"})"                 ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"F" }
      ],
      "transformations":[
        {"id":"seriesToColumns","options":{"byField":"ins"}},
        {
          "id":"organize",
          "options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true,"Time 6":true},"indexByName":{"Value #A":2,"Value #B":3,"Value #C":4,"Value #D":5,"Value #E":6,"Value #F":7,"ins":0,"ip":1},"renameByName":{"Value #A":"Up","Value #B":"Uptime","Value #C":"Alert Rules","Value #D":"Firing","Value #E":"Pending","Value #F":"Memory","ins":"Instance","ip":"IP"}}
        }
      ],
      "type":"table"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"fixedColor":"#b3596d","mode":"fixed"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":30,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":true,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":4,"x":20,"y":1},
      "id":203,
      "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vmalert_alerts_firing{ins=~\"$ins\"})","format":"time_series","intervalFactor":1,"legendFormat":"Firing Alerts","range":true,"refId":"A"}],
      "title":"Firing Alerts",
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":7},"id":210,"panels":[],"title":"Key Metrics","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"How long the VMAlert instance has been running",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]},"unit":"dtdurations"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":0,"y":8},
      "id":211,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min(vm_app_uptime_seconds{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Uptime","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total number of loaded alerting rules across selected instances and groups",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":3,"y":8},
      "id":212,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(vmalert_alerting_rules_last_evaluation_samples{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Alert Rules","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Number of alerts currently in firing state",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":6,"y":8},
      "id":213,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(vmalert_alerts_firing{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Firing","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Number of alerts currently in pending state",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":9,"y":8},
      "id":214,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(vmalert_alerts_pending{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Pending","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total number of loaded recording rules across selected instances and groups",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":12,"y":8},
      "id":215,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(vmalert_recording_rules_last_evaluation_samples{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Record Rules","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total errors from alerting and recording rules",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":15,"y":8},
      "id":216,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_alerting_rules_errors_total{ins=~\"$ins\"}[$__rate_interval])) + sum(increase(vmalert_recording_rules_errors_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"Errors","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Recording rules that produce no data (may be misconfigured)",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":18,"y":8},
      "id":217,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(vmalert_recording_rules_last_evaluation_samples{ins=~\"$ins\"} < 1)","format":"time_series","instant":true,"interval":"","legendFormat":"No Data","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Current resident memory usage of the process",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#b3596d","value":0}]},"unit":"bytes"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":21,"y":8},
      "id":218,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(process_resident_memory_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Memory","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"Shows the number of fired alerts by job.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":0,"y":11},
      "id":15,
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_alerts_fired_total{ins=~\"$ins\"}[$__rate_interval])) by(job)","interval":"","legendFormat":"{{job}}","range":true,"refId":"A"}],
      "title":"Alerts fired total ($ins)",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"Top $topk groups by evaluation duration. Shows groups that take the most of time during the evaluation across all instances.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"s"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":12,"y":11},
      "id":23,
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "editorMode":"code",
          "exemplar":false,
          "expr":"topk($topk, max(sum(\n    rate(vmalert_iteration_duration_seconds_sum{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])\n/\n    rate(vmalert_iteration_duration_seconds_count{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])\n) by(job, instance, group, file)) \nby(job, group, file))",
          "interval":"",
          "legendFormat":"({{job}}) {{group}}({{file}})",
          "range":true,
          "refId":"A"
        }
      ],
      "title":"Top $topk groups avg evaluation duration ($group)",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"Shows how many requests (executions) per second vmalert sends to the configured datasource.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":0,"y":19},
      "id":24,
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vmalert_execution_total{ins=~\"$ins\"}[$__rate_interval])) by (job)","interval":"","legendFormat":"{{job}}","range":true,"refId":"A"}],
      "title":"Rules execution rate ($ins)",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"Shows the error rate while executing configured rules. Non-zero value means there are some issues with existing rules. Check the logs to get more details.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":12,"y":19},
      "id":25,
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vmalert_execution_errors_total{ins=~\"$ins\"}[$__rate_interval])) by(job) > 0","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],
      "title":"Rules execution errors ($ins)",
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":27},"id":11,"panels":[],"title":"Logging","type":"row"},
    {
      "datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"fixedColor":"#346f36cc","mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"bars",
            "fillOpacity":100,
            "gradientMode":"none",
            "hideFrom":{"graph":false,"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":true,
            "stacking":{"group":"A","mode":"normal"},
            "thresholdsStyle":{"mode":"off"}
          },
          "decimals":0,
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"none"
        },
        "overrides":[]
      },
      "gridPos":{"h":5,"w":24,"x":0,"y":28},
      "id":219,
      "interval":"1s",
      "options":{"legend":{"calcs":["sum"],"displayMode":"table","placement":"right","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog ip:$ip unit:vmalert | stats count() as count","legendFormat":"syslog","queryType":"statsRange","refId":"A"}],
      "type":"timeseries"
    },
    {
      "datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
      "description":"syslog of grafana-server in $ins @ $ip",
      "gridPos":{"h":9,"w":24,"x":0,"y":33},
      "id":220,
      "maxDataPoints":100,
      "options":{"dedupStrategy":"none","detailsMode":"sidebar","enableInfiniteScrolling":true,"enableLogDetails":true,"fontSize":"small","prettifyLogMessage":false,"showCommonLabels":true,"showControls":true,"showLabels":true,"showTime":true,"sortOrder":"Descending","syntaxHighlighting":true,"wrapLogMessage":false},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog ip:$ip unit:vmalert ","queryType":"range","refId":"A"}],
      "type":"logs"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":42},
      "id":43,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"How much resident memory (RAM) the process is actually using, compared to its allowed container or system limit.\n\n- Good: Below 70% most of the time, maybe spiking a bit under load.\n- Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":93},
          "id":37,
          "links":[{"targetBlank":true,"title":"Profiling","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling"}],
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    max_over_time(process_resident_memory_bytes{ins=~\"$ins\"}[$__rate_interval])\n    /\n    vm_available_memory_bytes{ins=~\"$ins\"}\n) by(job)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Memory usage % ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the CPU usage percentage per vmalert instance. \nIf you think that usage is abnormal or unexpected pls file an issue and attach CPU profile if possible.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":93},
          "id":35,
          "links":[{"targetBlank":true,"title":"Profiling","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling"}],
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n  rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval]) \n  / \n  process_cpu_cores_available{ins=~\"$ins\"}\n) by(job)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"}],
          "title":"CPU usage %($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":125},
          "id":65,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    max_over_time(process_resident_memory_anon_bytes{ins=~\"$ins\"}[$__rate_interval])\n    /\n    vm_available_memory_bytes{ins=~\"$ins\"}\n) by(instance)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"RSS anonymous memory % usage",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the max number of CPU cores used by a `job` and the corresponding limit.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":125},
          "id":56,
          "links":[{"targetBlank":true,"title":"Profiling","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling"}],
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval])) by(job)","format":"time_series",             "interval":"","intervalFactor":1,"legendFormat":"{{job}}"        ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min(process_cpu_cores_available{ins=~\"$ins\"}) by(job)"                      ,"format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"limit ({{job}})","range":true,"refId":"B" }
          ],
          "title":"CPU usage ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Amount of used RSS memory\n\nIf you think that usage is abnormal or unexpected, please file an issue and attach memory profile if possible.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":133},
          "id":57,
          "links":[{"targetBlank":true,"title":"Profiling","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling"}],
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    max_over_time(process_resident_memory_bytes{ins=~\"$ins\"}[$__rate_interval])\n) by(job)","interval":"","legendFormat":"{{job}}","range":true,"refId":"A"}],
          "title":"Memory usage ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). Helps troubleshoot high CPU usage or throttling.\nLower is better.\n\n- waiting: The percentage of time at least one task in the process was ready to run (runnable) but couldn't get scheduled on the CPU.\n- stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time \u2014 a full CPU stall.\n\nIf there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5\u201310% and aligns with latency spikes or GC slowdowns.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":133},
          "id":66,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_waiting_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series",             "interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_stalled_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B" }
          ],
          "title":"CPU pressure",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\nLower is better.\n\n- waiting: Time fraction where at least one thread was blocked on memory.\n- stalled: Time fraction where every thread was blocked on memory (severe pressure).\n\nIf queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":141},
          "id":67,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_waiting_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series",             "interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_stalled_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B" }
          ],
          "title":"Memory pressure",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":141},
          "id":41,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_goroutines{ins=~\"$ins\"}) by(job)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Goroutines ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Panel shows the percentage of open file descriptors in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":3,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":149},
          "id":39,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    max_over_time(process_open_fds{ins=~\"$ins\"}[$__rate_interval])\n    /\n    process_max_fds{ins=~\"$ins\"}\n) by(job)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Open FDs usage % ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":0.1}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":149},
          "id":61,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{}[$__rate_interval])) by (job, instance, le))) by(job)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Go scheduling latency",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ ",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":157},
          "id":59,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(\n  rate(go_gc_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval])\n ) by(job)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"CPU spent on GC ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":165},
          "id":64,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(go_memstats_alloc_bytes_total{}[$__rate_interval])) by (job, instance)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Memory allocations rate",
          "type":"timeseries"
        }
      ],
      "title":"Resource usage",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":43},
      "id":62,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "fieldConfig":{
            "defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":true,"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},
            "overrides":[{"matcher":{"id":"byName","options":"Value"},"properties":[{"id":"custom.hideFrom.viz","value":true}]},{"matcher":{"id":"byName","options":"Time"},"properties":[{"id":"custom.hideFrom.viz","value":true}]}]
          },
          "gridPos":{"h":7,"w":12,"x":0,"y":94},
          "id":50,
          "options":{"cellHeight":"sm","filterable":true,"showHeader":true,"showSearch":true,"sortBy":[{"desc":true,"displayName":"job"}]},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(flag{is_set=\"true\", ins=~\"$ins\"}) by(job, instance, name, value)","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],
          "title":"Non-default flags",
          "transformations":[{"id":"groupBy","options":{"fields":{"instance":{"aggregations":["uniqueValues"],"operation":"aggregate"},"job":{"aggregations":[],"operation":"groupby"},"name":{"aggregations":[],"operation":"groupby"},"value":{"aggregations":[],"operation":"groupby"}}}}],
          "type":"table"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Missed evaluation means that group evaluation time takes longer than the configured evaluation interval. \nThis may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency for such groups. See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups\n\nIf rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries.\"",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"bars",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":7,"w":12,"x":12,"y":94},
          "id":58,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_iteration_missed_total{ins=~\"$ins\"}[$__rate_interval])) by(job, group, file) > 0","interval":"1m","legendFormat":"({{job}}) {{group}}({{file}})","range":true,"refId":"A"}],
          "title":"Missed evaluations ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMin":0,
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"stepAfter",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"none"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":125},
          "id":63,
          "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(changes(vm_app_start_timestamp{ins=~\"$ins\"}[$__rate_interval]) > 0) by(job)","format":"time_series","instant":false,"legendFormat":"{{job}}","refId":"A"}],
          "title":"Restarts ($job)",
          "type":"timeseries"
        }
      ],
      "title":"Troubleshooting",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":44},
      "id":17,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows top $topk current active (firing) alerting rules.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":95},
          "id":14,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"topk($topk, sum(vmalert_alerts_firing{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"}) by(job, group, file, alertname) > 0)","interval":"","legendFormat":"({{job}}) {{group}}.{{alertname}}({{file}})","range":true,"refId":"A"}],
          "title":"Top $topk active alerts ($group)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the events when rule execution resulted into an error. Check the logs for more details.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":95},
          "id":13,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_alerting_rules_errors_total{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) by(job, group, file, alertname) > 0","interval":"","legendFormat":"({{job}}) {{group}}.{{alertname}}({{file}})","range":true,"refId":"A"}],
          "title":"Errors ($group)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the current pending alerting rules per group.\nBy pending means the rule which remains active less than configured `for` parameter.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":103},
          "id":20,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Mean","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vmalert_alerts_pending{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"}) by(job, group, file, alertname) > 0","interval":"","legendFormat":"({{job}}) {{group}}.{{alertname}}({{file}})","range":true,"refId":"A"}],
          "title":"Pending ($group)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the error rate for the attempts to send alerts to Alertmanager. If not zero it means there issues on attempt to send notification to Alertmanager and some alerts may be not delivered properly. Check the logs for more details.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":103},
          "id":32,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"exemplar":false,"expr":"sum(rate(vmalert_alerts_send_errors_total{ins=~\"$ins\"}[$__rate_interval])) by(instance, addr) > 0","interval":"","legendFormat":"{{instance}} => {{addr}}","refId":"A"}],
          "title":"Errors rate to Alertmanager",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows how many alerts are sent to Alertmanager per second. Only active alerts are sent.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":5,"w":24,"x":0,"y":111},
          "id":26,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vmalert_alerts_sent_total{ins=~\"$ins\"}[$__rate_interval])) by(job, addr) > 0","interval":"","legendFormat":"{{job}} => {{addr}}","range":true,"refId":"A"}],
          "title":"Requests rate to Alertmanager by job ($group)",
          "type":"timeseries"
        }
      ],
      "title":"Alerting rules ($ins)",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":45},
      "id":28,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the top $topk recording rules which generate the most of [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples). Each generated sample is basically a time series which then ingested into configured remote storage. Rules with high numbers may cause the most pressure on the remote database and become a source of too high cardinality.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":96},
          "id":31,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            {
              "datasource":{"type":"prometheus","uid":"$ds"},
              "editorMode":"code",
              "exemplar":false,
              "expr":"topk($topk, \n  max(\n    sum(vmalert_recording_rules_last_evaluation_samples{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"}) by(job, instance, group, file, recording) > 0\n  ) by(job, group, file, recording)\n)",
              "interval":"",
              "legendFormat":"({{job}}) {{group}}.{{recording}}({{file}})",
              "range":true,
              "refId":"A"
            }
          ],
          "title":"Top $topk rules by produced samples ($group)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the rules which do not produce any [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) during the evaluation. Usually it means that such rules are misconfigured, since they give no output during the evaluation.\nPlease check if rule's expression is correct and it is working as expected.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":true,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":96},
          "id":33,
          "options":{"legend":{"calcs":["lastNotNull","max","mean"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"count(vmalert_recording_rules_last_evaluation_samples{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"} < 1) by(job, group, file, recording)","interval":"","legendFormat":"({{job}}) {{group}}.{{recording}}({{file}})","range":true,"refId":"A"}],
          "title":"Rules with 0 produced samples ($group)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":4,"w":24,"x":0,"y":104},
          "id":30,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_recording_rules_errors_total{ins=~\"$ins\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) by(job, group, file, recording) > 0","interval":"","legendFormat":"({{job}}) {{group}}.{{recording}}({{file}})","range":true,"refId":"A"}],
          "title":"Errors ($group)",
          "type":"timeseries"
        }
      ],
      "title":"Recording rules ($ins)",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":46},
      "id":55,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":97},
          "id":52,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmalert_remotewrite_sent_rows_total{ins=~\"$ins\"}[$__rate_interval])) by(job)","legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Datapoints send rate ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the number of datapoints dropped by vmalert while sending to the configured remote write URL. vmalert performs up to 5 retries before dropping the data. Check vmalert's error logs for the specific error message.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":97},
          "id":53,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmalert_remotewrite_dropped_rows_total{ins=~\"$ins\"}[$__rate_interval])) by(job) > 0","legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Datapoints drop rate ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows current number of established connections to remote write endpoints.\n\n",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":105},
          "id":54,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(max_over_time(vmalert_remotewrite_conns{ins=~\"$ins\"}[$__rate_interval])) by(job)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Connections ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the global rate for number of written bytes via remote write connections.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"decbytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":105},
          "id":60,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vmalert_remotewrite_conn_bytes_written_total{ins=~\"$ins\"}[$__rate_interval])) by(job) > 0","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Bytes write rate ($ins)",
          "type":"timeseries"
        }
      ],
      "title":"Remote write",
      "type":"row"
    }
  ],
  "preload":false,
  "refresh":"",
  "schemaVersion":42,
  "tags":["Pigsty","INFRA","VICTORIA","VMALERT","INSTANCE"],
  "templating":{
    "list":[
      {"current":{},"hide":2,"includeAll":false,"name":"ds","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},
      {"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{cls=\"vmalert\"},ins)","includeAll":false,"name":"ins","options":[],"query":{"qryType":1,"query":"label_values(vm_app_version{cls=\"vmalert\"},ins)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"},
      {"current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{ins=\"$ins\"},ip)","hide":2,"name":"ip","options":[],"query":{"qryType":1,"query":"label_values(vm_app_version{ins=\"$ins\"},ip)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"},
      {"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vmalert_iteration_total{ins=~\"$ins\"},file)","includeAll":true,"multi":true,"name":"file","options":[],"query":{"query":"label_values(vmalert_iteration_total{ins=~\"$ins\"},file)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"},
      {"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vmalert_iteration_total{ins=~\"$ins\"}, group)","includeAll":true,"multi":true,"name":"group","options":[],"query":{"query":"label_values(vmalert_iteration_total{ins=~\"$ins\"}, group)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","type":"query"},
      {"current":{},"includeAll":false,"name":"topk","options":[{"selected":true,"text":"5","value":"5"},{"selected":false,"text":"10","value":"10"},{"selected":false,"text":"20","value":"20"},{"selected":false,"text":"30","value":"30"},{"selected":false,"text":"40","value":"40"},{"selected":false,"text":"50","value":"50"}],"query":"5, 10, 20, 30, 40, 50","type":"custom"},
      {"baseFilters":[],"datasource":{"type":"prometheus","uid":"${ds}"},"filters":[],"name":"filter","type":"adhoc"}
    ]
  },
  "time":{"from":"now-1h","to":"now"},
  "timepicker":{},
  "timezone":"",
  "title":"Victoria Alert",
  "uid":"vmalert-instance",
  "version":1,
  "weekStart":""
}