{
  "annotations":{
    "list":[
      {"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},
      {"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(ALERTS{ins=~\"$ins\", alertgroup=\"vmsingle\",alertstate=\"firing\",show_at=\"dashboard\"}) by(alertname)","hide":false,"iconColor":"red","name":"alerts","titleFormat":"{{alertname}}"},
      {"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"hide":true,"iconColor":"dark-blue","name":"version","target":{"expr":"sum(vm_app_version{ins=~\"$ins\"}) by(version) unless (sum(vm_app_version{ins=~\"$ins\"} offset $__interval) by(version))","interval":"","refId":"Anno"},"textFormat":"{{version}}","titleFormat":"Version change"}
    ]
  },
  "author":"Ruohang Feng (rh@vonng.com)",
  "description":"Victoria Metrics Instance",
  "editable":true,
  "fiscalYearStartMonth":0,
  "graphTooltip":1,
  "id":null,
  "license":"https://pigsty.io/docs/about/license/",
  "links":[{"asDropdown":true,"icon":"external link","includeVars":false,"keepTime":true,"tags":["Pigsty","INFRA"],"targetBlank":false,"title":"INFRA","tooltip":"","type":"dashboards","url":""}],
  "panels":[
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":200,"panels":[],"title":"Overview","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]}},"overrides":[{"matcher":{"id":"byName","options":"IP"},"properties":[{"id":"links","value":[{"title":"Node Instance : $ip","url":"/d/node-instance?var-id=${ip}&${__url_time_range}"}]}]}]},
      "gridPos":{"h":6,"w":5,"x":0,"y":1},
      "id":201,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"/.*/","values":true},"showPercentChange":false,"text":{"titleSize":16,"valueSize":16},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"vm_app_version{ins=\"$ins\"}","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],
      "transformations":[
        {
          "id":"organize",
          "options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"cls":true,"job":true,"short_version":false,"version":true},"includeByName":{},"indexByName":{"Time":5,"Value":9,"__name__":6,"cls":3,"ins":0,"instance":1,"ip":2,"job":7,"short_version":4,"version":8},"renameByName":{"Value":"","cls":"Cluster","ins":"Instance","instance":"Address","ip":"IP","job":"","short_version":"Version"}}
        }
      ],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"All VictoriaMetrics instances",
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"custom":{"align":"center","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#e3e3e3e0","value":0}]}},
        "overrides":[
          { "matcher":{"id":"byName","options":"Instance"}  ,"properties":[{"id":"links","value":[{"title":"VMetrics Instance ${__data.fields.Instance}","url":"/d/vmetrics-instance?var-ins=${__data.fields.Instance}&${__url_time_range}"}]},{"id":"custom.width"}]                                                                                                                                                    },
          { "matcher":{"id":"byName","options":"IP"}        ,"properties":[{"id":"custom.width","value":110},{"id":"links","value":[{"title":"Node Instance : ${__data.fields.IP}","url":"/d/node-instance?var-id=${__data.fields.IP}&${__url_time_range}"}]}]                                                                                                                                                           },
          { "matcher":{"id":"byName","options":"Up"}        ,"properties":[{"id":"custom.width","value":50},{"id":"mappings","value":[{"options":{"0":{"color":"#cc4637d9","index":0,"text":"Dead"},"1":{"color":"#346f36cc","index":1,"text":"Alive"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"type":"color-background"}}]                                                                               },
          { "matcher":{"id":"byName","options":"Uptime"}    ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"dtdurations"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"red","value":0},{"color":"#fcdb72","value":300},{"color":"#346f36cc","value":3600}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                                                        },
          { "matcher":{"id":"byName","options":"Datapoints"},"properties":[{"id":"custom.width","value":90},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":1000000},{"color":"#fcdb72","value":100000000},{"color":"#f79f64","value":1000000000}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]          },
          { "matcher":{"id":"byName","options":"Active"}    ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":10000},{"color":"#fcdb72","value":100000},{"color":"#f79f64","value":1000000}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                  },
          { "matcher":{"id":"byName","options":"Ingest"}    ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":100},{"color":"#fcdb72","value":100000},{"color":"#f79f64","value":1000000}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                    },
          { "matcher":{"id":"byName","options":"Disk"}      ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"bytes"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":1073741824},{"color":"#fcdb72","value":107374182400},{"color":"#f79f64","value":1099511627776}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}] },
          { "matcher":{"id":"byName","options":"Memory"}    ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"bytes"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":1073741824},{"color":"#fcdb72","value":8589934592},{"color":"#f79f64","value":17179869184}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]     }
        ]
      },
      "gridPos":{"h":6,"w":15,"x":5,"y":1},
      "id":202,
      "options":{"cellHeight":"sm","showHeader":true,"sortBy":[{"desc":false,"displayName":"Instance"}]},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min by (ins,ip) (up{cls=\"vmetrics\"})"                                             ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"A" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min by (ins) (vm_app_uptime_seconds{cls=\"vmetrics\"})"                             ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"B" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (vm_rows{cls=\"vmetrics\", type!~\"indexdb.*\"})"                      ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"C" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (vm_cache_entries{cls=\"vmetrics\", type=\"storage/hour_metric_ids\"})","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"D" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (rate(vm_rows_inserted_total{cls=\"vmetrics\"}[$__rate_interval]))"    ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"E" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (vm_data_size_bytes{cls=\"vmetrics\"})"                                ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"F" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (process_resident_memory_bytes{cls=\"vmetrics\"})"                     ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"G" }
      ],
      "transformations":[
        {"id":"seriesToColumns","options":{"byField":"ins"}},
        {
          "id":"organize",
          "options":{
            "excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true,"Time 6":true,"Time 7":true},
            "indexByName":{"Value #A":2,"Value #B":3,"Value #C":4,"Value #D":5,"Value #E":6,"Value #F":7,"Value #G":8,"ins":0,"ip":1},
            "renameByName":{"Value #A":"Up","Value #B":"Uptime","Value #C":"Datapoints","Value #D":"Active","Value #E":"Ingest","Value #F":"Disk","Value #G":"Memory","ins":"Instance","ip":"IP"}
          }
        }
      ],
      "type":"table"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"fixedColor":"#a64d4d","mode":"fixed"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":30,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":true,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":4,"x":20,"y":1},
      "id":203,
      "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_rows_inserted_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","intervalFactor":1,"legendFormat":"Ingestion Rate","range":true,"refId":"A"}],
      "title":"Ingestion Rate",
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":7},"id":210,"panels":[],"title":"Key Metrics","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"How long the VictoriaMetrics instance has been running",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]},"unit":"dtdurations"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":0,"y":8},
      "id":211,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min(vm_app_uptime_seconds{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Uptime","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total number of datapoints stored in the database",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":3,"y":8},
      "id":212,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(vm_rows{ins=~\"$ins\", type!~\"indexdb.*\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Data Points","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Number of active time series with recent data points",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":6,"y":8},
      "id":213,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(vm_cache_entries{ins=~\"$ins\", type=\"storage/hour_metric_ids\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Active Series","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Current data ingestion rate in samples per second",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":9,"y":8},
      "id":214,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vm_rows_inserted_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"Ingest Rate","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total disk space used by VictoriaMetrics data",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]},"unit":"bytes"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":12,"y":8},
      "id":215,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(vm_data_size_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Disk Usage","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Current resident memory usage of the process",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]},"unit":"bytes"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":15,"y":8},
      "id":216,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(process_resident_memory_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Memory","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"CPU utilization of the VictoriaMetrics process",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]},"unit":"percentunit"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":18,"y":8},
      "id":217,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"CPU Usage","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Rate of incoming read/query requests",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#a64d4d","value":0}]},"unit":"reqps"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":21,"y":8},
      "id":218,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vm_http_requests_total{ins=~\"$ins\", path=~\"/api/v1/query.*\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"Read Requests","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"How many datapoints are inserted into storage per second",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":0,"y":11},
      "id":106,
      "options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_rows_inserted_total{ins=~\"$ins\"}[$__rate_interval])) by (type, instance) > 0","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A"}],
      "title":"Datapoints ingestion rate",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":12,"y":11},
      "id":12,
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_http_requests_total{ins=~\"$ins\"}[$__rate_interval])) by (path, instance) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}} - {{path}}","range":true,"refId":"A"}],
      "title":"Requests rate",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown. \n\nSee following link for details:",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":0,"y":19},
      "id":51,
      "links":[{"targetBlank":true,"title":"troubleshooting","url":"https://docs.victoriametrics.com/victoriametrics/troubleshooting/"}],
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"vm_cache_entries{ins=~\"$ins\", type=\"storage/hour_metric_ids\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
      "title":"Active time series",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"s"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":12,"y":19},
      "id":22,
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vm_request_duration_seconds{ins=~\"$ins\", quantile=\"0.99\"}) by (instance, path) > 0","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}} - {{path}}","range":true,"refId":"A"}],
      "title":"Query duration 0.99 quantile",
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":27},"id":24,"panels":[],"title":"Logging","type":"row"},
    {
      "datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"fixedColor":"#346f36cc","mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"bars",
            "fillOpacity":100,
            "gradientMode":"none",
            "hideFrom":{"graph":false,"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":true,
            "stacking":{"group":"A","mode":"normal"},
            "thresholdsStyle":{"mode":"off"}
          },
          "decimals":0,
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"none"
        },
        "overrides":[]
      },
      "gridPos":{"h":4,"w":24,"x":0,"y":28},
      "id":219,
      "interval":"1s",
      "options":{"legend":{"calcs":["sum"],"displayMode":"table","placement":"right","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog ip:$ip unit:vmetrics | stats count() as count","legendFormat":"syslog","queryType":"statsRange","refId":"A"}],
      "type":"timeseries"
    },
    {
      "datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
      "description":"syslog of grafana-server in $ins @ $ip",
      "gridPos":{"h":8,"w":24,"x":0,"y":32},
      "id":220,
      "maxDataPoints":100,
      "options":{"dedupStrategy":"none","detailsMode":"sidebar","enableInfiniteScrolling":true,"enableLogDetails":true,"fontSize":"small","prettifyLogMessage":false,"showCommonLabels":true,"showControls":true,"showLabels":true,"showTime":true,"sortOrder":"Descending","syntaxHighlighting":true,"wrapLogMessage":false},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog ip:$ip unit:vmetrics ","queryType":"range","refId":"A"}],
      "type":"logs"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":0,"y":40},
      "id":35,
      "options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vm_http_request_errors_total{ins=~\"$ins\"}[$__rate_interval])) by (instance, path) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}} - {{path}}","range":true,"refId":"A"}],
      "title":"Requests error rate",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"bars",
            "fillOpacity":100,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"auto",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":12,"y":40},
      "id":110,
      "options":{"legend":{"calcs":["lastNotNull","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_log_messages_total{ins=~\"$ins\", level!=\"info\"}[$__rate_interval])) by (instance, level, location) > 0","interval":"5m","legendFormat":"{{instance}} - {{level}}: {{location}}","range":true,"refId":"A"}],
      "title":"Logging rate",
      "type":"timeseries"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":48},
      "id":46,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"How much resident memory (RAM) the process is actually using, compared to its allowed container or system limit.\n\n- Good: Below 70% most of the time, maybe spiking a bit under load.\n- Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":327},
          "id":112,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    max_over_time(process_resident_memory_bytes{ins=~\"$ins\"}[$__rate_interval])\n    /\n    vm_available_memory_bytes{ins=~\"$ins\"}\n) by(instance)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"RSS memory % usage",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":327},
          "id":114,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{ins=~\"$ins\"}\n) by(instance)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"CPU % usage",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":335},
          "id":123,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    max_over_time(process_resident_memory_anon_bytes{ins=~\"$ins\"}[$__rate_interval])\n    /\n    vm_available_memory_bytes{ins=~\"$ins\"}\n) by(instance)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"RSS anonymous memory % usage",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"CPU cores used by instance",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byName","options":"Limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}}]}]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":335},
          "id":57,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code",                 "expr":"rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval])","format":"time_series","interval":"",             "intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min(process_cpu_cores_available{ins=~\"$ins\"})"                 ,"format":"time_series","interval":"","hide":false,"intervalFactor":1,"legendFormat":"Limit"       ,"range":true,"refId":"B" }
          ],
          "title":"CPU",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":343},
          "id":44,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code",                 "expr":"sum(go_memstats_sys_bytes{ins=~\"$ins\"}) by (instance) + sum(vm_cache_size_bytes{ins=~\"$ins\"}) by (instance)"       ,"format":"time_series","hide":false,              "intervalFactor":1,"legendFormat":"{{instance}} - requested from system","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code",                 "expr":"sum(go_memstats_heap_inuse_bytes{ins=~\"$ins\"}) by (instance) + sum(vm_cache_size_bytes{ins=~\"$ins\"}) by (instance)","format":"time_series","hide":false,              "intervalFactor":1,"legendFormat":"{{instance}} - heap inuse"           ,"range":true,"refId":"B" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code",                 "expr":"sum(go_memstats_stack_inuse_bytes{ins=~\"$ins\"}) by (instance)"                                                       ,"format":"time_series","hide":false,              "intervalFactor":1,"legendFormat":"{{instance}} - stack inuse"          ,"range":true,"refId":"C" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code",                 "expr":"sum(process_resident_memory_bytes{ins=~\"$ins\"}) by (instance)"                                                       ,"format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}} - resident"             ,"range":true,"refId":"D" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(process_resident_memory_anon_bytes{ins=~\"$ins\"}) by (instance)"                                                  ,"format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}} - resident anonymous"   ,"range":true,"refId":"E" }
          ],
          "title":"Memory usage",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). Helps troubleshoot high CPU usage or throttling.\nLower is better.\n\n- waiting: The percentage of time at least one task in the process was ready to run (runnable) but couldn't get scheduled on the CPU.\n- stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time \u2014 a full CPU stall.\n\nIf there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5\u201310% and aligns with latency spikes or GC slowdowns.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":343},
          "id":134,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_waiting_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series",             "interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_stalled_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B" }
          ],
          "title":"CPU pressure",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\nLower is better.\n\n- waiting: Time fraction where at least one thread was blocked on memory.\n- stalled: Time fraction where every thread was blocked on memory (severe pressure).\n\nIf queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.\n\nSee also major page faults rate panel in Troubleshooting section if this metric continued to be high.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":351},
          "id":135,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_waiting_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series",             "interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_stalled_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B" }
          ],
          "title":"Memory pressure",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the number of bytes read/write from the storage layer.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[{"matcher":{"id":"byRegexp","options":"/.*read/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":351},
          "id":76,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_storage_read_bytes_total{ins=~\"$ins\"}[$__rate_interval])) by (instance)"   ,"format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}} - read" ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_storage_written_bytes_total{ins=~\"$ins\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}} - write","range":true,"refId":"B" }
          ],
          "title":"Disk writes/reads",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":359},
          "id":47,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_goroutines{ins=~\"$ins\"}) by(instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Goroutines",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the number of read/write syscalls such as read, pread, write, pwrite.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byRegexp","options":"/.*read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":359},
          "id":124,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_read_syscalls_total{ins=~\"$ins\"}[$__rate_interval])) by (instance)" ,"format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}} - read calls" ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_write_syscalls_total{ins=~\"$ins\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}} - write calls","range":true,"refId":"B" }
          ],
          "title":"Disk write/read calls",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":367},
          "id":48,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(process_num_threads{ins=~\"$ins\"}) by(instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Threads",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\nThe lower the better.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":367},
          "id":136,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_io_waiting_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series",             "interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_io_stalled_seconds_total{}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B" }
          ],
          "title":"IO pressure",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":375},
          "id":49,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_tcplistener_accepts_total{ins=~\"$ins\"}[$__rate_interval])) by(instance)","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"TCP connections rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":375},
          "id":37,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_tcplistener_conns{ins=~\"$ins\"}) by(instance)","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"TCP connections",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":2,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[{"matcher":{"id":"byName","options":"max"},"properties":[{"id":"color","value":{"fixedColor":"#C4162A","mode":"fixed"}}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":383},
          "id":75,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max_over_time(process_open_fds{ins=~\"$ins\"}[$__rate_interval])\n/\nprocess_max_fds{ins=~\"$ins\"}","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Open FDs",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ ",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":383},
          "id":125,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(\n  rate(go_gc_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval])\n ) by(instance)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"CPU spent on GC",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows network usage by VM:\n* Writes show traffic sent to clients\n* Reads show traffic received from clients",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"decbits"
            },
            "overrides":[{"matcher":{"id":"byRegexp","options":"/read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":391},
          "id":127,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_tcplistener_read_bytes_total{ins=~\"$ins\"}[$__rate_interval])) by(name) * 8 > 0"   ,"format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"read via {{name}}" ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_tcplistener_written_bytes_total{ins=~\"$ins\"}[$__rate_interval])) by(name) * 8 > 0","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"write via {{name}}","range":true,"refId":"B" }
          ],
          "title":"Network usage",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":391},
          "id":133,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(go_memstats_alloc_bytes_total{}[$__rate_interval])) by (job, instance)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Memory allocations rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":0.1}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":399},
          "id":128,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{}[$__rate_interval])) by (job, instance, le))","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Go scheduling latency",
          "type":"timeseries"
        }
      ],
      "title":"Resource usage",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":49},
      "id":71,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the rate and total number of new series created over last 24h.\n\nHigh [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#cardinality) \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":154},
          "id":66,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_new_timeseries_created_total{ins=~\"$ins\"}[$__rate_interval])) by (instance)","interval":"","legendFormat":"{{instance}} - churn rate"         ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(increase(vm_new_timeseries_created_total{ins=~\"$ins\"}[24h])) by (instance)"         ,"interval":"","legendFormat":"{{instance}} - new series over 24h","range":true,"refId":"B" }
          ],
          "title":"Churn rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"The percentage of [slow inserts](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \n\nThe lower the better. \n\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \n* Ingestion of completely new, not seen before time series;\n* Not enough memory to maintain big enough caches for the current workload.\n\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series). \n\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line+area"}
              },
              "decimals":2,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":0},{"color":"red","value":0.1}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":154},
          "id":68,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(\n    rate(vm_slow_row_inserts_total{}[$__rate_interval]) \n    / rate(vm_rows_added_to_storage_total{}[$__rate_interval])\n) by (instance)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Slow inserts %",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Merge assist happens when storage can't keep up with merging parts. This is usually a sign of overload for storage.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"bars",
                "fillOpacity":100,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":162},
          "id":116,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vm_assisted_merges_total{ins=~\"$ins\"}[$__rate_interval])) by(instance, type) > 0","format":"time_series","interval":"5m","intervalFactor":1,"legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A"}],
          "title":"Assisted merges",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Slow queries rate according to `search.logSlowQueryDuration` flag, which is `5s` by default.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":162},
          "id":60,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_slow_queries_total{ins=~\"$ins\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Slow queries rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":9,"w":12,"x":0,"y":170},
          "id":90,
          "options":{"legend":{"calcs":["lastNotNull","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"vm_cache_size_bytes{ins=~\"$ins\"} / vm_cache_size_max_bytes{ins=~\"$ins\"}","interval":"","legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A"}],
          "title":"Cache usage % by type",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows cache miss ratio. Lower is better.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "max":1,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":9,"w":12,"x":12,"y":170},
          "id":118,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"(\n    rate(vm_cache_misses_total{ins=~\"$ins\"}[$__rate_interval])\n    /\n    rate(vm_cache_requests_total{ins=~\"$ins\"}[$__rate_interval])\n) > 0","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A"}],
          "title":"Cache miss ratio",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows how many samples were ignored or dropped on insertion due to various reasons:\n* timestamp out of retention period or timestamp in future;\n* invalid metric name;\n* exceeding limit for labels length or number;\n* dropped by [relabeling configuration](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#relabeling).",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":179},
          "id":131,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(increase(vm_rows_ignored_total{ins=~\"$ins\"}[1h])) by (instance, reason)"  ,"hide":false,                "interval":"","legendFormat":"{{instance}} - {{reason}}","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code",                "expr":"sum(increase(vm_relabel_metrics_dropped_total{ins=~\"$ins\"}[1h])) by(instance)","hide":false,"instant":false,              "legendFormat":"{{instance}} - relabeling","range":true,"refId":"B" }
          ],
          "title":"Samples dropped for last 1h",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.\n\nShould be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn't set.\n\nDeduplication happens in two places:\n* `type=\"merge\"` - during [background merges](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage). Deduplication during merges is permanent.\n* `type=\"select\"` - during [read queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data). Deduplication happens in-flight and has no permanent effect.\n\nThe more samples need to be deduplicated, the higher will be resource usage.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":179},
          "id":132,
          "links":[{"targetBlank":true,"title":"Readonly mode","url":"https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode"}],
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_deduplicated_samples_total{ins=~\"$ins\"}[$__rate_interval])) by(instance, type) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Deduplication rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Flags explicitly set to non-default values",
          "fieldConfig":{
            "defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":true,"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},
            "overrides":[{"matcher":{"id":"byName","options":"Time"},"properties":[{"id":"custom.hideFrom.viz","value":true}]},{"matcher":{"id":"byName","options":"Value"},"properties":[{"id":"custom.hideFrom.viz","value":true}]},{"matcher":{"id":"byName","options":"job"},"properties":[{"id":"custom.hideFrom.viz","value":true}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":187},
          "id":126,
          "options":{"cellHeight":"sm","filterable":true,"showHeader":true,"showSearch":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(flag{is_set=\"true\", ins=~\"$ins\"}) by(job, instance, name, value)","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],
          "title":"Non-default flags",
          "type":"table"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMin":0,
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"stepAfter",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"none"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":187},
          "id":129,
          "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(changes(vm_app_start_timestamp{ins=~\"$ins\"}[$__rate_interval]) > 0) by(job)","format":"time_series","instant":false,"legendFormat":"{{job}}","refId":"A"}],
          "title":"Restarts ($job)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"${ds}"},
          "description":"Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:\n- insufficient physical memory\n- active use of swap space\n- file mapping access\n\nIt may cause significant performance slowdown if sustained high.\n\nPossible workaround:\n- Extend RAM to scale up\n- Reduce swap usage by adjusting vm.swappiness",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[{"targetBlank":true,"title":"Drilldown","url":"/d/wNf0q_kZk?viewPanel=140&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$ins&${__url_time_range}"}],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":195},
          "id":137,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"sum(rate(process_major_pagefaults_total{ins=~\"$ins\"}[$__rate_interval])) by (job)","legendFormat":"__auto","range":true,"refId":"A"}],
          "title":"Major page faults rate",
          "type":"timeseries"
        }
      ],
      "title":"Troubleshooting",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":50},
      "id":14,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"How many datapoints are inserted into storage per second",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":458},
          "id":10,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_rows_inserted_total{ins=~\"$ins\"}[$__rate_interval])) by (instance, type) > 0","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A"}],
          "title":"Datapoints ingestion rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows the approx time needed to reach 100% of allowed disk capacity based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":458},
          "id":73,
          "options":{"legend":{"calcs":["mean","lastNotNull","min"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            {
              "datasource":{"type":"prometheus","uid":"$ds"},
              "editorMode":"code",
              "expr":"(vm_free_disk_space_bytes{ins=~\"$ins\"}-vm_free_disk_space_limit_bytes{ins=~\"$ins\"}) \n/ \nignoring(path) (\n    (rate(vm_rows_added_to_storage_total{ins=~\"$ins\"}[1d]) - \n        sum(rate(vm_deduplicated_samples_total{ins=~\"$ins\"}[1d])) without(type)) * \n    (\n        sum(vm_data_size_bytes{ins=~\"$ins\", type!~\"indexdb.*\"}) without(type) /\n        sum(vm_rows{ins=~\"$ins\", type!~\"indexdb.*\"}) without(type)\n    )\n    +\n    rate(vm_new_timeseries_created_total{ins=~\"$ins\"}[1d]) * \n    (\n        sum(vm_data_size_bytes{ins=~\"$ins\", type=\"indexdb/file\"}) without(type) /\n        sum(vm_rows{ins=~\"$ins\", type=\"indexdb/file\"}) without(type)\n    )\n) > 0",
              "format":"time_series",
              "hide":false,
              "interval":"",
              "intervalFactor":1,
              "legendFormat":"{{instance}}",
              "range":true,
              "refId":"A"
            }
          ],
          "title":"Storage full ETA",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows amount of on-disk space occupied by data points and the remaining disk space at `-storageDataPath`. Calculation accounts for -storage.minFreeDiskSpaceBytes",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":466},
          "id":53,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_data_size_bytes{ins=~\"$ins\", type!~\"indexdb.*\"}) by (instance)"              ,"format":"time_series","interval":"",             "intervalFactor":1,"legendFormat":"{{instance}} - Used (datapoints)","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},                    "expr":"vm_free_disk_space_bytes{ins=~\"$ins\"} - vm_free_disk_space_limit_bytes{ins=~\"$ins\"}","format":"time_series","interval":"",             "intervalFactor":1,"legendFormat":"{{instance}} - Free"             ,"range":true,"refId":"B" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_data_size_bytes{ins=~\"$ins\", type=~\"indexdb.*\"}) by (instance)"              ,"format":"time_series","interval":"","hide":false,"intervalFactor":1,"legendFormat":"{{instance}} - Used (index)"     ,"range":true,"refId":"C" }
          ],
          "title":"Disk space usage - datapoints",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byName","options":"pending index entries"},"properties":[{"id":"unit","value":"none"},{"id":"decimals","value":3}]}]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":466},
          "id":34,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"expr":"vm_pending_rows{ins=~\"$ins\", type=\"storage\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}} - pending datapoints"   ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"expr":"vm_pending_rows{ins=~\"$ins\", type=\"indexdb\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}} - pending index entries","range":true,"refId":"B" }
          ],
          "title":"Pending datapoints",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows how many datapoints are in the storage and what is average disk usage per datapoint.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byName","options":"bytes-per-datapoint"},"properties":[{"id":"unit","value":"bytes"},{"id":"decimals","value":2}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":474},
          "id":30,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_rows{ins=~\"$ins\", type!~\"indexdb.*\"}) by (instance)"                                                        ,"format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}} - total datapoints"   ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_data_size_bytes{ins=~\"$ins\"}) by (instance)\n/ sum(vm_rows{ins=~\"$ins\", type!~\"indexdb.*\"}) by (instance)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}} - bytes-per-datapoint","range":true,"refId":"B" }
          ],
          "title":"Datapoints",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$ds"},
          "description":"The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage).\nIncrease in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage (hot data)\n* `storage/big` - small parts gradually merged into bigger parts (cold data)",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":474},
          "id":36,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_parts{ins=~\"$ins\", type=~\"indexdb.*\"}) by (instance, type)","format":"time_series",             "intervalFactor":1,"legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vm_last_partition_parts{ins=~\"$ins\"}) by(instance, type)"       ,"format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}} - {{type}}","range":true,"refId":"B" }
          ],
          "title":"LSM parts max by type",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps and how many series were ignored on insertion or during scrape due to too long label names, values and excessive labels count.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":482},
          "id":58,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vm_rows_ignored_total{ins=~\"$ins\"}[1h])) by (instance, reason)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}} - {{reason}}","range":true,"refId":"A"}],
          "title":"Rows ignored for last 1h",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"The number of on-going merges in storage nodes. It is expected to have high numbers for `storage/small` metric.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":482},
          "id":62,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_active_merges{ins=~\"$ins\"}) by(instance, type)","legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A"}],
          "title":"Active merges",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. However, these two actions are separated and controlled via different concurrency limiters. The `max` on this panel can't be changed and always equal to number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU.\n\n",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byName","options":"max"},"properties":[{"id":"color","value":{"fixedColor":"#C4162A","mode":"fixed"}}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":490},
          "id":59,
          "options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max_over_time(vm_concurrent_insert_capacity{ins=~\"$ins\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}} - max"    ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_concurrent_insert_current{ins=~\"$ins\"}) by (instance)"               ,"format":"time_series",              "intervalFactor":1,"legendFormat":"{{instance}} - current","range":true,"refId":"B" }
          ],
          "title":"Concurrent flushes on disk",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"The number of rows merged per second by storage nodes.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":490},
          "id":64,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_rows_merged_total{ins=~\"$ins\"}[$__rate_interval])) by(instance, type)","legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A"}],
          "title":"Merge speed",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"99th percentile of number of series read per query.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":2,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":498},
          "id":99,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Series read per query",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per queried series.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":2,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":498},
          "id":103,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Datapoints read per series",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per query.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":2,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":506},
          "id":122,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Datapoints read per query",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scanner per query.\n\nThis number can exceed number of RowsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":2,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":506},
          "id":105,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Datapoints scanned per query",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Shows number of existing [snapshots](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots).\n\nRecently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.\n\nIt is recommended deleting old snapshots when they are no longer needed to free up disk space.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":514},
          "id":130,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"vm_snapshots{ins=~\"$ins\"}","interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Number of snapshots",
          "type":"timeseries"
        }
      ],
      "title":"Storage",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":51},
      "id":138,
      "panels":[
        {"gridPos":{"h":2,"w":17,"x":0,"y":523},"id":139,"options":{"code":{"language":"plaintext","showLineNumbers":false,"showMiniMap":false},"content":"Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance, or a single instance in cases like memory breakdown.","mode":"markdown"},"pluginVersion":"12.3.0","transparent":true,"type":"text"},
        {
          "datasource":{"type":"prometheus","uid":"${ds}"},
          "description":"Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:\n- insufficient physical memory\n- active use of swap space\n- file mapping access\n\nIt may cause significant performance slowdown if sustained high.\n\nPossible workaround:\n- Extend RAM to scale up\n- Reduce swap usage by adjusting vm.swappiness",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":525},
          "id":140,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"sum(rate(process_major_pagefaults_total{ins=~\"$ins\"}[$__rate_interval])) by (job,instance)","legendFormat":"{{instance}} ({{job}})","range":true,"refId":"A"}],
          "title":"Major page faults rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"${ds}"},
          "description":"This panel breaks down memory usage by type. It\u2019s intended for inspecting a single instance.\n\nHow to use:\n- Start from the high-level RSS panel.\n- Identify an instance with unexpected or abnormal memory growth.\n- Filter to that instance to inspect the detailed breakdown.\n\nInterpretation\n- A steadily rising Go Heap usually indicates a memory leak. Collect pprof memory profile.\n- A growing Go Stack commonly points to a goroutine leak.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"normal"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":525},
          "id":141,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"# go other\nmax(\n    go_memstats_gc_sys_bytes{ins=~\"$ins\"} +\n    go_memstats_mspan_sys_bytes{ins=~\"$ins\"} +\n    go_memstats_other_sys_bytes{ins=~\"$ins\"} +\n    go_memstats_mcache_sys_bytes{ins=~\"$ins\"}\n)",                             "legendFormat":"Go: Other"        ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"# go stack\nmax(go_memstats_stack_inuse_bytes{ins=~\"$ins\"})"                                                                                                                                                          ,"hide":false,"instant":false,"legendFormat":"Go: Stack"        ,"range":true,"refId":"B" },
            { "datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"#go heap inuse\nmax(\n    go_memstats_heap_inuse_bytes{ins=~\"$ins\"}\n    +\n    (\n        go_memstats_heap_idle_bytes{ins=~\"$ins\"}\n        -\n        go_memstats_heap_released_bytes{ins=~\"$ins\"}\n    )\n)"   ,"hide":false,"instant":false,"legendFormat":"Go: Heap"         ,"range":true,"refId":"C" },
            { "datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"# go heap reserved\nmax(go_memstats_heap_released_bytes{ins=~\"$ins\"})"                                                                                                                                                ,"hide":false,"instant":false,"legendFormat":"Go: Heap Reserved","range":true,"refId":"D" },
            { "datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"# vm cache\nmax(sum(vm_cache_size_bytes{ins=~\"$ins\"}) by (job, instance))"                                                                                                                                            ,"hide":false,"instant":false,"legendFormat":"Mmap: VM Cache"   ,"range":true,"refId":"E" },
            { "datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"# file pages\nmax(process_resident_memory_file_bytes{ins=~\"$ins\"})"                                                                                                                                                   ,"hide":false,"instant":false,"legendFormat":"File cache"       ,"range":true,"refId":"F" }
          ],
          "title":"Memory usage breakdown",
          "type":"timeseries"
        }
      ],
      "title":"Drilldown",
      "type":"row"
    }
  ],
  "preload":false,
  "refresh":"",
  "schemaVersion":42,
  "tags":["Pigsty","INFRA","VICTORIA","VMETRICS","INSTANCE"],
  "templating":{
    "list":[
      {"current":{},"hide":2,"includeAll":false,"name":"ds","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},
      {"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{cls=\"vmetrics\"},ins)","includeAll":false,"name":"ins","options":[],"query":{"qryType":1,"query":"label_values(vm_app_version{cls=\"vmetrics\"},ins)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"},
      {"current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{ins=\"$ins\"},ip)","hide":2,"name":"ip","options":[],"query":{"qryType":1,"query":"label_values(vm_app_version{ins=\"$ins\"},ip)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"},
      {"baseFilters":[],"datasource":{"type":"prometheus","uid":"$ds"},"filters":[],"name":"adhoc","type":"adhoc"}
    ]
  },
  "time":{"from":"now-3h","to":"now"},
  "timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},
  "timezone":"",
  "title":"Victoria Metrics",
  "uid":"vmetrics-instance",
  "version":1,
  "weekStart":""
}