{
  "annotations":{
    "list":[
      {"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},
      {"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(vm_app_version{ins=~\"$ins\"}) by(short_version) unless (sum(vm_app_version{ins=~\"$ins\"} offset $__interval) by(short_version))","hide":true,"iconColor":"blue","name":"version change","textFormat":"{{short_version}}","titleFormat":"Version change"},
      {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"enable":true,"expr":"sum(changes(vm_app_start_timestamp{ins=~\"$ins\"}[$__interval])) by(job)","hide":true,"iconColor":"orange","name":"restarts","textFormat":"{{instance}} restarted"},
      {"datasource":{"type":"prometheus","uid":"${ds}"},"enable":true,"filter":{"exclude":false,"ids":[75]},"hide":false,"iconColor":"#57f26d1c","name":"gc","target":{"expr":"go_memstats_last_gc_time_seconds{ins=~\"$ins\"} * 1000","interval":"","refId":"Anno"},"textFormat":"GC event","useValueForTime":true}
    ]
  },
  "author":"Ruohang Feng (rh@vonng.com)",
  "description":"VictoriaTraces Single Node Dashboard",
  "editable":true,
  "fiscalYearStartMonth":0,
  "graphTooltip":0,
  "id":null,
  "license":"https://pigsty.io/docs/about/license/",
  "links":[{"asDropdown":true,"icon":"external link","includeVars":false,"keepTime":true,"tags":["Pigsty","INFRA"],"targetBlank":false,"title":"INFRA","tooltip":"","type":"dashboards","url":""}],
  "panels":[
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":200,"panels":[],"title":"Overview","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"IP"},"properties":[{"id":"links","value":[{"title":"Node Instance : $ip","url":"/d/node-instance?var-id=${ip}&${__url_time_range}"}]}]}]},
      "gridPos":{"h":6,"w":5,"x":0,"y":1},
      "id":201,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"/.*/","values":true},"showPercentChange":false,"text":{"titleSize":16,"valueSize":16},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"vm_app_version{ins=\"$ins\"}","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],
      "title":"",
      "transformations":[
        {
          "id":"organize",
          "options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"cls":true,"job":true,"short_version":false,"version":true},"includeByName":{},"indexByName":{"Time":5,"Value":9,"__name__":6,"cls":3,"ins":0,"instance":1,"ip":2,"job":7,"short_version":4,"version":8},"renameByName":{"Value":"","cls":"Cluster","ins":"Instance","instance":"Address","ip":"IP","job":"","short_version":"Version"}}
        }
      ],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"All VictoriaTraces instances",
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"custom":{"align":"center","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#e3e3e3e0","value":null}]}},
        "overrides":[
          {"matcher":{"id":"byName","options":"Instance"},"properties":[{"id":"links","value":[{"title":"VTraces Instance ${__data.fields.Instance}","url":"/d/vtraces-instance?var-ins=${__data.fields.Instance}&${__url_time_range}"}]},{"id":"custom.width"}]},
          {"matcher":{"id":"byName","options":"IP"},"properties":[{"id":"custom.width","value":110},{"id":"links","value":[{"title":"Node Instance : ${__data.fields.IP}","url":"/d/node-instance?var-id=${__data.fields.IP}&${__url_time_range}"}]}]},
          {"matcher":{"id":"byName","options":"Up"},"properties":[{"id":"custom.width","value":50},{"id":"mappings","value":[{"options":{"0":{"color":"#cc4637d9","index":0,"text":"Dead"},"1":{"color":"#346f36cc","index":1,"text":"Alive"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"type":"color-background"}}]},
          {"matcher":{"id":"byName","options":"Uptime"},"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"dtdurations"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"red","value":null},{"color":"#fcdb72","value":300},{"color":"#346f36cc","value":3600}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]},
          {"matcher":{"id":"byName","options":"Traces"},"properties":[{"id":"custom.width","value":90},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":null},{"color":"#346f36cc","value":1000000},{"color":"#fcdb72","value":100000000},{"color":"#f79f64","value":1000000000}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]},
          {"matcher":{"id":"byName","options":"Ingest"},"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":null},{"color":"#346f36cc","value":100},{"color":"#fcdb72","value":10000},{"color":"#f79f64","value":100000}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]},
          {"matcher":{"id":"byName","options":"Disk"},"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"bytes"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":null},{"color":"#346f36cc","value":1073741824},{"color":"#fcdb72","value":107374182400},{"color":"#f79f64","value":1099511627776}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]},
          {"matcher":{"id":"byName","options":"Memory"},"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"bytes"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":null},{"color":"#346f36cc","value":1073741824},{"color":"#fcdb72","value":8589934592},{"color":"#f79f64","value":17179869184}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]}
        ]
      },
      "gridPos":{"h":6,"w":15,"x":5,"y":1},
      "id":202,
      "options":{"cellHeight":"sm","showHeader":true,"sortBy":[{"desc":false,"displayName":"Instance"}]},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min by (ins,ip) (up{cls=\"vtraces\"})"                                         ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"A" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min by (ins) (vm_app_uptime_seconds{cls=\"vtraces\"})"                         ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"B" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (vt_storage_rows{cls=\"vtraces\"})"                               ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"C" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (rate(vt_rows_ingested_total{cls=\"vtraces\"}[$__rate_interval]))","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"D" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (vt_data_size_bytes{cls=\"vtraces\"})"                            ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"E" },
        { "datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (process_resident_memory_bytes{cls=\"vtraces\"})"                 ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"F" }
      ],
      "transformations":[
        {"id":"seriesToColumns","options":{"byField":"ins"}},
        {
          "id":"organize",
          "options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true,"Time 6":true},"indexByName":{"Value #A":2,"Value #B":3,"Value #C":4,"Value #D":5,"Value #E":6,"Value #F":7,"ins":0,"ip":1},"renameByName":{"Value #A":"Up","Value #B":"Uptime","Value #C":"Traces","Value #D":"Ingest","Value #E":"Disk","Value #F":"Memory","ins":"Instance","ip":"IP"}}
        }
      ],
      "type":"table"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"fixedColor":"#c4895c","mode":"fixed"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":30,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":true,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":4,"x":20,"y":1},
      "id":203,
      "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vt_rows_ingested_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","intervalFactor":1,"legendFormat":"Ingestion Rate","range":true,"refId":"A"}],
      "title":"Ingestion Rate",
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":7},"id":210,"panels":[],"title":"Key Metrics","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"How long the VictoriaTraces instance has been running",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]},"unit":"dtdurations"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":0,"y":8},
      "id":211,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min(vm_app_uptime_seconds{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Uptime","refId":"A"}],
      "title":"",
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total number of trace spans stored",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":3,"y":8},
      "id":212,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(vt_storage_rows{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Traces","refId":"A"}],
      "title":"",
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Current trace ingestion rate in spans per second",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":6,"y":8},
      "id":213,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vt_rows_ingested_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"Ingest Rate","refId":"A"}],
      "title":"",
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Data ingestion rate in bytes per second",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]},"unit":"binBps"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":9,"y":8},
      "id":214,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vt_bytes_ingested_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"Bytes Ingested","refId":"A"}],
      "title":"",
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total disk space used by VictoriaTraces data",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]},"unit":"bytes"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":12,"y":8},
      "id":215,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(vt_data_size_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Disk Usage","refId":"A"}],
      "title":"",
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Current resident memory usage of the process",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]},"unit":"bytes"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":15,"y":8},
      "id":216,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(process_resident_memory_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Memory","refId":"A"}],
      "title":"",
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Rate of incoming query requests",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]},"unit":"reqps"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":18,"y":8},
      "id":217,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vt_select_requests_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"Query Rate","refId":"A"}],
      "title":"",
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Number of active traces",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#c4895c","value":null}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":21,"y":8},
      "id":218,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(vt_active_traces{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Active Traces","refId":"A"}],
      "title":"",
      "type":"stat"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":11},
      "id":16,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Total amount of trace spans in the storage.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},
          "gridPos":{"h":4,"w":4,"x":0,"y":12},
          "id":133,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vt_storage_rows{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"total","refId":"A"}],
          "title":"Total trace spans",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"The total number of trace spans ingested over the past 24 hours.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},
          "gridPos":{"h":4,"w":5,"x":4,"y":12},
          "id":134,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vt_rows_ingested_total{ins=~\"$ins\"}[24h]))","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"total","refId":"A"}],
          "title":"Ingested spans 24h",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Average ingestion rate of trace spans.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},
          "gridPos":{"h":4,"w":5,"x":9,"y":12},
          "id":135,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vt_rows_ingested_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"total","refId":"A"}],
          "title":"Insert req/s",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Total amount of used disk space.\nAccounts for all compressed trace spans and index size.\n\nSee how to [control disk usage](https://docs.victoriametrics.com/victoriatraces/#retention-by-disk-space-usage).",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"bytes"},"overrides":[]},
          "gridPos":{"h":4,"w":5,"x":14,"y":12},
          "id":136,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vt_data_size_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"total","refId":"A"}],
          "title":"Disk space usage",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Integer number of CPU cores available to the application. This value is automatically rounded down from fractional CPU quotas. For optimal performance, fractional CPU units should be avoided. See the [best practices](https://docs.victoriametrics.com/victoriametrics/bestpractices/#kubernetes) documentation for more details.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},
          "gridPos":{"h":4,"w":5,"x":19,"y":12},
          "id":137,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vm_available_cpu_cores{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"total","refId":"A"}],
          "title":"Available CPU",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"How many spans are ingested per second.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},
          "gridPos":{"h":4,"w":4,"x":0,"y":16},
          "id":144,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vt_storage_rows{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"total","refId":"A"}],
          "title":"Ingested spans/s",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"The cumulative number of trace spans ingested over the last 24h. \n\nThe size is calculated before compression.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"bytes"},"overrides":[]},
          "gridPos":{"h":4,"w":5,"x":4,"y":16},
          "id":139,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vt_bytes_ingested_total{ins=~\"$ins\"}[24h]))","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"total","refId":"A"}],
          "title":"Ingested bytes 24h",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Rate of HTTP read requests.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},
          "gridPos":{"h":4,"w":5,"x":9,"y":16},
          "id":140,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vt_http_requests_total{ins=~\"$ins\", path=~\"/select/.*\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"total","refId":"A"}],
          "title":"Read req/s",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"The ratio between original data size and compressed data stored on disk. This metric excludes indexdb size. \n\nThe ratio can go up or down as the system performs automatic maintenance and applies retention policies. For examples:\n- Background merges: [Merges](https://docs.victoriametrics.com/victoriatraces/#forced-merge) improve compression by combining data into larger, more efficiently compressed blocks\n- Retention policies: When old data is deleted due to retention settings, the ratio changes as different time periods have varying compression characteristics\n\n",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"none"},"overrides":[]},
          "gridPos":{"h":4,"w":5,"x":14,"y":16},
          "id":141,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":" sum(vt_uncompressed_data_size_bytes{ins=~\"$ins\"}) / sum(vt_compressed_data_size_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"{{instance}}","refId":"A"}],
          "title":"Compression ratio",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Total system memory available to the application. This represents the system or container's memory capacity or limit, not the currently free memory.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"bytes"},"overrides":[]},
          "gridPos":{"h":4,"w":5,"x":19,"y":16},
          "id":142,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vm_available_memory_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"total","refId":"A"}],
          "title":"Available memory",
          "type":"stat"
        }
      ],
      "title":"Stats",
      "type":"row"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":20},"id":18,"panels":[],"title":"Overview","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Ingestion rate in number of trace spans and bytes per second.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[{"matcher":{"id":"byRegexp","options":"/.*(bytes)/"},"properties":[{"id":"custom.axisPlacement","value":"right"},{"id":"unit","value":"bytes"}]}]
      },
      "gridPos":{"h":8,"w":12,"x":0,"y":21},
      "id":2,
      "options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"sum(rate(vt_rows_ingested_total{ins=~\"$ins\"}[$__rate_interval])) by (type) > 0" ,"format":"time_series","hide":false,                "intervalFactor":1,"legendFormat":"{{type}}"        ,"range":true,"refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(rate(vt_bytes_ingested_total{ins=~\"$ins\"}[$__rate_interval])) by (type) > 0",                       "hide":false,"instant":false,                   "legendFormat":"{{type}} (bytes)","range":true,"refId":"B" }
      ],
      "title":"Trace spans ingestion rate",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"* `*` - unsupported query path\n* `/insert` - [inserts](https://docs.victoriametrics.com/victoriatraces/data-ingestion/)\n* `/select` - [reads](https://docs.victoriametrics.com/victoriatraces/querying/)\n* `/metrics` - scraping of system metrics",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "noValue":"0",
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":12,"y":21},
      "id":14,
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(rate(vt_http_requests_total{ins=~\"$ins\"}[$__rate_interval])) by (path) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],
      "title":"Requests rate",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"$ds"},
      "description":"* `*` - unsupported query path\n* `/insert` - [inserts](https://docs.victoriametrics.com/victoriatraces/data-ingestion/)\n* `/select` - [reads](https://docs.victoriametrics.com/victoriatraces/querying/)\n* `/metrics` - scraping of system metrics",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":0,"y":29},
      "id":69,
      "options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vm_http_request_errors_total{ins=~\"$ins\"}[$__rate_interval])) by (path)","format":"time_series",             "interval":"","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A" },
        { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","exemplar":false,"expr":"sum(rate(vt_http_errors_total{ins=~\"$ins\"}[$__rate_interval])) by (path)"        ,"format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"B" }
      ],
      "title":"Requests error rate",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"99th percentile response time for VictoriaTraces HTTP endpoints, grouped by instance and path. This means 99% of requests are faster than this value. **Lower numbers are better**, as they indicate faster responses and fewer slow requests.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "noValue":"0",
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"s"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":12,"y":29},
      "id":66,
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vt_http_request_duration_seconds{ins=~\"$ins\", quantile=\"0.99\"}) by (path) > 0","format":"time_series","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],
      "title":"Request duration p99",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Amount of on-disk space occupied by all the data stored in the storage.\n\nSee how to [control disk usage](https://docs.victoriametrics.com/victoriatraces/#retention-by-disk-space-usage).",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "axisSoftMax":1,
            "axisSoftMin":0,
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "links":[],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"bytes"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":12,"x":0,"y":37},
      "id":6,
      "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vt_total_disk_space_bytes{ins=~\"$ins\"}) by (instance)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
      "title":"Disk space usage",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Rate of VictoriaTraces' own application log messages (errors, warnings, debug)",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"bars",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"auto",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
        },
        "overrides":[{"matcher":{"id":"byRegexp","options":".*warn:.*"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":".*error:.*"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]}]
      },
      "gridPos":{"h":8,"w":12,"x":12,"y":37},
      "id":67,
      "options":{"legend":{"calcs":["lastNotNull","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_log_messages_total{ins=~\"$ins\", level!=\"info\"}[$__rate_interval])) by (instance, level, location)","interval":"5m","legendFormat":"{{instance}} - {{level}}: {{location}}","range":true,"refId":"A"}],
      "title":"Logging rate",
      "type":"timeseries"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":45},
      "id":28,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Percentage of physical RAM used compared to the memory limit. If this percentage is high, check the `RSS` anonymous vs resident ratio panel for more details.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"yellow","value":0.8},{"color":"red","value":0.9}]},"unit":"percentunit"},"overrides":[]},
          "gridPos":{"h":5,"w":7,"x":0,"y":78},
          "id":79,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    max_over_time(process_resident_memory_bytes{ins=~\"$ins\"}[$__rate_interval])\n    /\n    vm_available_memory_bytes{ins=~\"$ins\"}\n)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"{{instance}}","refId":"A"}],
          "title":"RSS % of memory limit",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"How much memory (RAM) the VictoriaTraces process is actually using, compared to its allowed container or system limit. See 'Memory Usage' panel for a detailed breakdown.\n\n- Good: Below 70% most of the time, maybe spiking a bit under load.\n- Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "max":1,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":10,"w":17,"x":7,"y":78},
          "id":38,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    max_over_time(process_resident_memory_bytes{ins=~\"$ins\"}[$__rate_interval])\n    /\n    vm_available_memory_bytes{ins=~\"$ins\"}\n) by (instance)","interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Physical memory usage ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Portion of RAM that CANNOT be reclaimed without swapping. If both the `RSS`-to-limit percentage and this ratio are high, the process is at high risk of an `OOM` kill.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"percentunit"},"overrides":[]},
          "gridPos":{"h":5,"w":7,"x":0,"y":83},
          "id":78,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(process_resident_memory_anon_bytes{ins=~\"$ins\"}\n/\nprocess_resident_memory_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"{{instance}}","refId":"A"}],
          "title":"Anonymous / Resident ratio",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"- Anonymous resident memory: Private memory allocated to the application that **cannot** be reclaimed by the kernel. Refer to the [Check/profile](https://docs.victoriametrics.com/victoriatraces/#profiling) Go heap section for troubleshooting.\n- File-backed resident memory: Memory mapped from files, which can be safely reclaimed. Increases during querying. Correlate with `I/O` panels for further analysis.\n- Shared resident memory: Typically negligible. Large spikes may indicate unexpected shared memory consumers.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":88},
          "id":40,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"sum(process_resident_memory_anon_bytes{ins=~\"$ins\"})"  ,"format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"anonymous resident memory"  ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(process_resident_memory_file_bytes{ins=~\"$ins\"})"  ,"format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"file-backed resident memory","range":true,"refId":"B" },
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"sum(process_resident_memory_shared_bytes{ins=~\"$ins\"})","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"shared resident memory"     ,"range":true,"refId":"C" }
          ],
          "title":"Memory usage ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "max":1,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":88},
          "id":118,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(\n    rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{ins=~\"$ins\"}\n) by(instance)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"CPU % usage ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Lower is better, e.g. 20% means the process was delayed by memory pressure 20% of the time. See [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n- waiting: Time fraction where at least one thread was blocked on memory.\n- stalled: Time fraction where every thread was blocked on memory (severe pressure).\n\nIf queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "max":1,
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":0.2}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":96},
          "id":73,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"sum(rate(process_pressure_memory_waiting_seconds_total{}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_stalled_seconds_total{}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B" }
          ],
          "title":"Memory pressure",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Helps troubleshoot high CPU usage or throttling:\n\n- waiting: The percentage of time at least one task in the VictoriaTraces process was ready to run (runnable) but couldn't get scheduled on the CPU.\n- stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time \u2014 a full CPU stall.\n\nIf there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5\u201310% and aligns with latency spikes or GC slowdowns.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMax":1,
                "axisSoftMin":0,
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":96},
          "id":72,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_waiting_seconds_total{}[$__rate_interval])) by (instance)","format":"time_series",             "interval":"","intervalFactor":2,"legendFormat":"{{instance}}: waiting","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"sum(rate(process_pressure_cpu_stalled_seconds_total{}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}: stalled","range":true,"refId":"B" }
          ],
          "title":"CPU pressure",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Number of read/write calls application makes:\n\n- read call: Number of read*()-family system calls your process has issued since start. Each call can move 1 byte or megabytes, cached or uncached.\n- write call: Number of write*()-family system calls (including write, pwrite, writev, etc.) made by the process.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byName","options":"read calls"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":104},
          "id":56,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"max(rate(process_io_read_syscalls_total{ins=~\"$ins\"}[$__rate_interval]))" ,"format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"read calls" ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(rate(process_io_write_syscalls_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"write calls","range":true,"refId":"B" }
          ],
          "title":"Read/Write syscalls ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Percentage of open file descriptors (files, sockets, pipes, etc.,) compared to the limit set in the OS. Reaching the limit of open files can cause various issues and must be prevented.\n\nSee [how to change limits](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a).",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":2,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[{"matcher":{"id":"byName","options":"max"},"properties":[{"id":"color","value":{"fixedColor":"#C4162A","mode":"fixed"}}]}]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":104},
          "id":46,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max_over_time(process_open_fds{ins=~\"$ins\"}[$__rate_interval])\n/\nprocess_max_fds{ins=~\"$ins\"}","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Open FDs % usage ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Measure the actual bytes read from and written to disk by the process:\n\n- read: physical bytes the kernel actually pulled from the storage device on behalf of the process (after checking page-cache).\n- write: physical bytes the kernel ultimately wrote to the storage device for the process (after combining, caching, or delaying writes).",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[{"matcher":{"id":"byName","options":"read"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":112},
          "id":52,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(rate(process_io_storage_read_bytes_total{ins=~\"$ins\"}[$__rate_interval]))"   ,"format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"read" ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"max(rate(process_io_storage_written_bytes_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"write","range":true,"refId":"C" }
          ],
          "title":"Disk writes/reads ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). The lower the better.\n\n- waiting: at least one runnable thread blocked on block-`I/O` (disk, NVMe, network-storage) while others could still make progress.\n- stalled: all non-idle threads simultaneously waiting on `I/O`; no useful user code ran during these periods \u2192 true `I/O` thrashing.\n\nIf stalled > 0 while querying, it's recommended to increase queue depth on NVMe, raise blk-mq budgets, or relax cgroup I/O limits.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":112},
          "id":74,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"max(rate(process_pressure_io_waiting_seconds_total{}[$__rate_interval])) by (instance)","format":"time_series",             "interval":"","intervalFactor":2,"legendFormat":"{{instance}}: waiting","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(rate(process_pressure_io_stalled_seconds_total{}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}: stalled","range":true,"refId":"B" }
          ],
          "title":"IO pressure",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Current number of active TCP connections to VictoriaTraces. This metric helps monitor connection pool usage and identify potential connection leaks. High values may indicate clients not properly closing connections or connection pooling issues. Monitor for gradual increases that could lead to resource exhaustion.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":120},
          "id":60,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(vm_tcplistener_conns{ins=~\"$ins\"}) by (instance)","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"TCP connections ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":120},
          "id":50,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(go_goroutines{ins=~\"$ins\"}) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Goroutines ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Rate of incoming TCP connections accepted by VictoriaTraces. This metric indicates network activity and client connection patterns. Sudden spikes may indicate increased load or potential DDoS attacks. Sustained high rates should be correlated with resource usage to ensure adequate capacity.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":128},
          "id":58,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(rate(vm_tcplistener_accepts_total{ins=~\"$ins\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"TCP connections rate ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":128},
          "id":54,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(process_num_threads{ins=~\"$ins\"}) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Threads ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing `GOGC` to higher values. Increasing `GOGC` value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ ",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":136},
          "id":119,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(\n  rate(go_gc_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__rate_interval])\n ) by(instance)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"CPU spent on GC ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":0.1}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":136},
          "id":61,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{}[$__rate_interval])) by (instance, le))) by (instance)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Go scheduling latency",
          "type":"timeseries"
        }
      ],
      "title":"Resource usage",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":46},
      "id":68,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. \n\nNormally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMin":0,
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"stepAfter",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"none"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":145},
          "id":62,
          "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(changes(vm_app_start_timestamp{ins=~\"$ins\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"legendFormat":"{{instance}}","refId":"A"}],
          "title":"Restarts",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"The number of new [streams](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#stream-fields) created over the last 24 hours. Lower rate is better.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineStyle":{"fill":"solid"},
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":145},
          "id":26,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(increase(vt_streams_created_total{ins=~\"$ins\"}[1d])) by (instance)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Streams churn rate 24h ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Flags explicitly set to non-default values",
          "fieldConfig":{
            "defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},
            "overrides":[{"matcher":{"id":"byName","options":"Time"},"properties":[{"id":"custom.hideFrom.viz","value":true}]},{"matcher":{"id":"byName","options":"Value"},"properties":[{"id":"custom.hideFrom.viz","value":true}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":153},
          "id":70,
          "options":{"cellHeight":"sm","showHeader":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(flag{is_set=\"true\", ins=~\"$ins\"}) by(job, instance, name, value)","format":"table","instant":true,"legendFormat":"{{name}}={{value}}","range":false,"refId":"A"}],
          "title":"Non-default flags",
          "transformations":[{"id":"groupBy","options":{"fields":{"instance":{"aggregations":["uniqueValues"],"operation":"aggregate"},"job":{"aggregations":[],"operation":"groupby"},"name":{"aggregations":[],"operation":"groupby"},"value":{"aggregations":[],"operation":"groupby"}}}}],
          "type":"table"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Number of trace spans ignored or dropped on insertion due to the following reasons:\n* Timestamp out of the retention period or in the future\n* Number of fields per entry exceeded\n* Line too long\n\nIf this occurs, check the VictoriaTraces log for details.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":153},
          "id":71,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"sum(increase(vt_rows_dropped_total{ins=~\"$ins\"}[1h])) by (reason) ","hide":false,                "interval":"","legendFormat":"{{reason}}"   ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code",                "expr":"sum(increase(vt_too_long_lines_skipped_total{ins=~\"$ins\"}[1h]))"   ,"hide":false,"instant":false,              "legendFormat":"line_too_long","range":true,"refId":"B" }
          ],
          "title":"Trace spans dropped for last 1h",
          "type":"timeseries"
        }
      ],
      "title":"Troubleshooting",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":47},
      "id":82,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Total number of time-based (daily) partitions in storage. The number typically grows over time as new data arrives and is partitioned by time periods. \n\nExcessive partition counts may indicate retention policy issues or very high data ingestion rates that could impact query performance.",
          "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},
          "gridPos":{"h":8,"w":3,"x":0,"y":162},
          "id":88,
          "maxDataPoints":100,
          "options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vt_partitions{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"{{instance}}","refId":"A"}],
          "title":"Partition Count",
          "type":"stat"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Number of storage parts (data files) in each tier. More parts mean fragmentation; fewer parts suggest successful merging. High part counts may slow queries and trigger background merge operations.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":9,"x":3,"y":162},
          "id":84,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","exemplar":true,"expr":"max(vt_storage_parts{ins=~\"$ins\"}) by(type)","hide":false,"interval":"","legendFormat":"{{type}}","range":true,"refId":"A"}],
          "title":"Part count max by type ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Disk space usage and limits for VictoriaTraces storage. Tracks current data usage against the configured retention limit or available disk space.\n\nThe red line indicates the space retention limit. When usage approaches this limit, older data will be automatically deleted. If the space retention limit (`-retention.maxDiskSpaceUsageBytes`) is not specified, the red line represents the maximum disk space. In that case, the storage will switch to read-only mode when the limit is reached.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMax":1,
                "axisSoftMin":0,
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"line"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":1}]},
              "unit":"percentunit"
            },
            "overrides":[{"matcher":{"id":"byName","options":"max space retention of instances"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}},{"id":"custom.insertNulls","value":3600000}]}]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":162},
          "id":110,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"${ds}"}        ,"editorMode":"code","exemplar":true,"expr":"max(max_over_time(vt_max_disk_space_usage_bytes{ins=~\"$ins\"}[$__rate_interval])) by (job)\n/\nmax(vt_total_disk_space_bytes{ins=~\"$ins\"}) by (job) > 0","hide":false,"interval":"","legendFormat":"max space retention of instances","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"max(max_over_time(vt_data_size_bytes{ins=~\"$ins\"}[$__rate_interval])) by (job)\n/\nmax(vt_total_disk_space_bytes{ins=~\"$ins\"}) by (job)"               ,"hide":false,"interval":"","legendFormat":"{{job}}"                         ,"range":true,"refId":"B" }
          ],
          "title":"Disk ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"${ds}"},
          "description":"Number of storage merge operations by type (sum across instances). Merges compact smaller parts into larger ones; bursts are normal after activity spikes.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":170},
          "id":112,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"sum(increase(vt_merges_total{ins=~\"$ins\"}[$__rate_interval])) by(type)","hide":false,"interval":"$__rate_interval","legendFormat":"{{type}}","range":true,"refId":"A"}],
          "title":"Merge events",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"${ds}"},
          "description":"99th percentile duration of merge operations by storage type. Merge operations combine smaller storage parts into larger ones for optimization. \n\nNormal merge durations vary by storage type and data volume. Consistently high durations may indicate storage performance issues, high write load, or insufficient resources for background operations. Monitor for trends that could impact overall system performance.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":170},
          "id":86,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"max(vl_merge_duration_seconds{ins=~\"$ins\", quantile=\"0.9\"}) by (type) > 0","hide":false,"interval":"","legendFormat":"{{type}}","range":true,"refId":"A"}],
          "title":"Merge duration p99 ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"${ds}"},
          "description":"99th percentile of data volume processed during merge operations by storage type. \n\nThis metric indicates the scale of background storage optimization activities. Larger merge sizes generally improve storage efficiency but require more resources. Consistently high values may indicate heavy write loads or large storage parts that need optimization. Monitor correlation with merge duration for performance insights.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":24,"x":0,"y":178},
          "id":109,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"max(vl_merge_bytes{ins=~\"$ins\", quantile=\"0.99\"}) by (type) > 0","hide":false,"interval":"","legendFormat":"{{type}}","range":true,"refId":"A"}],
          "title":"Merge bytes p99 ($ins)",
          "type":"timeseries"
        }
      ],
      "title":"Storage",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":48},
      "id":83,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Rate of incoming trace span insertion requests by endpoint path. This metric tracks the ingestion load on VictoriaTraces, including different insertion methods and protocols.\n\nHigher rates indicate increased trace span ingestion activity. Monitor for sudden spikes that might indicate new sources, application deployments, or potential issues requiring capacity planning. Sustained high rates may require scaling ingestion capacity.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byRegexp","options":"/.*(bytes)/"},"properties":[{"id":"custom.axisPlacement","value":"right"},{"id":"unit","value":"bytes"}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":187},
          "id":113,
          "options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(rate(vt_rows_ingested_total{ins=~\"$ins\"}[$__rate_interval])) by (type) > 0" ,"format":"time_series","hide":false,                "intervalFactor":1,"legendFormat":"{{type}}"        ,"range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"sum(rate(vt_bytes_ingested_total{ins=~\"$ins\"}[$__rate_interval])) by (type) > 0",                       "hide":false,"instant":false,                   "legendFormat":"{{type}} (bytes)","range":true,"refId":"B" }
          ],
          "title":"Trace spans ingestion rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Rate of incoming query requests by endpoint path. This metric tracks the query load on VictoriaTraces, including different query interfaces and internal operations. \n\nHigher rates indicate increased query activity from users or applications. Monitor for sudden spikes that might indicate new dashboards, automated queries, or potential performance issues. Sustained high rates may require scaling query processing capacity.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":187},
          "id":117,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vt_http_requests_total{ins=~\"$ins\",path=~\"^/(internal/)?insert.*\"}[$__rate_interval])) by (path) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],
          "title":"Request rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"99th percentile of insert operation duration. This represents the time it takes for 99% of insert operations to complete. High values indicate slow ingestion performance that could affect overall system throughput. Spikes may suggest storage bottlenecks, resource contention, or inefficient data processing.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":195},
          "id":103,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vt_http_request_duration_seconds{ins=~\"$ins\", quantile=\"0.99\", path=~\"^/(internal/)?insert.*\"}) by (path) > 0","format":"time_series","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],
          "title":"Insert duration p99",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"99th percentile response time for VictoriaTraces HTTP endpoints, grouped by instance and path. This means 99% of requests are faster than this value. **Lower numbers are better**, as they indicate faster responses and fewer slow requests.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":195},
          "id":116,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vt_http_request_duration_seconds{ins=~\"$ins\", quantile=\"0.99\", path=~\"^/(internal/)?insert.*\"}) by (path) > 0","format":"time_series","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],
          "title":"Request duration p99 ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Number of active message processors handling trace span ingestion. These processors parse and process incoming trace spans before storage. The count typically correlates with the ingestion load.\n\nIf this number is unusually high or inflated (which is rare), check memory usage. It may indicate a heavy, concurrently ingestion load or processing bottlenecks that could benefit from performance tuning.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"opacity",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "decimals":0,
              "links":[],
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":203},
          "id":97,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"max(max_over_time(vt_insert_processors_count{ins=~\"$ins\"}[$__rate_interval])) by (instance)","format":"time_series",             "interval":"",                "intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(max_over_time(vm_concurrent_insert_capacity{ins=~\"$ins\"}[$__rate_interval]))"           ,                       "hide":false,              "instant":false,                   "legendFormat":"soft limit"  ,"range":true,"refId":"B" }
          ],
          "title":"Message processors ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Number of trace span row waiting to be written to storage, categorized by type. Pending rows indicate temporary queuing during ingestion. Consistently high values may suggest storage write bottlenecks or insufficient write capacity.\n\nPending rows are flushed in two ways:\n\n- After a specific time period (typically 1 second)\n- When the pending row size exceeds a threshold (typically 1.75 MB)",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":203},
          "id":104,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(vt_pending_rows{ins=~\"$ins\"}) by (type)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{type}}","range":true,"refId":"A"}],
          "title":"Pending rows ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Number of concurrent insert operations has reached the configured limit: -maxConcurrentInserts (default: 2x CPU cores\n",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"opacity",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byName","options":"limit"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":211},
          "id":143,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(increase(vm_concurrent_insert_limit_reached_total[$__rate_interval])) by (instance)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Concurrent insert limit reached ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Number of insert requests that timed out while waiting for available concurrency slots. This indicates sustained ingestion overload beyond configured limits.\n\nHigh values suggest:\n- Insert queue is consistently full\n- Insert requests waiting too long for execution slots\n- System under sustained heavy ingestion load\n- Need for horizontal scaling or ingestion optimization\n\nCombined with `Insert concurrency limit reached`, provides complete picture of ingestion rejection patterns.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"vis":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":1}]},
              "unit":"reqps"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":211},
          "id":131,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(rate(vm_concurrent_insert_limit_timeout_total{ins=~\"$ins\"}[$__rate_interval])) by (instance)","interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Insert timeouts ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"99th percentile duration of background flush operations by type. High values may indicate disk pressure or heavy ingestion. Correlate with `I/O` panels.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":219},
          "id":115,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(vt_insert_flush_duration_seconds{ins=~\"$ins\", quantile=\"0.99\"}) by (type) > 0","format":"time_series","intervalFactor":1,"legendFormat":"{{type}}","range":true,"refId":"A"}],
          "title":"Flush duration p99",
          "type":"timeseries"
        }
      ],
      "title":"Ingestion",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":49},
      "id":81,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Rate of incoming query requests by endpoint path. This metric tracks the query load on VictoriaTraces, including different query interfaces and internal operations. \n\nHigher rates indicate increased query activity from users or applications. Monitor for sudden spikes that might indicate new dashboards, automated queries, or potential performance issues. Sustained high rates may require scaling query processing capacity.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":228},
          "id":102,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(rate(vt_http_requests_total{ins=~\"$ins\",path=~\"^/(internal/)?select.*\"}[$__rate_interval])) by (path) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],
          "title":"Query rate",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"99th percentile of query execution duration. This represents the time it takes for 99% of queries to complete:\n\n- High values indicate slow query performance that affects user experience. \n- Spikes may suggest complex queries, resource contention, or inefficient indexes. Monitor for trends that could indicate degrading performance.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":228},
          "id":108,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(vt_http_request_duration_seconds{ins=~\"$ins\", quantile=\"0.99\", path=~\"^/(internal/)?select.*\"}) by (path) > 0","format":"time_series","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],
          "title":"Query duration p99",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"$ds"},
          "description":"Number of concurrent select (query) operations compared to the configured limit.\n\nHigh utilization near the limit may indicate query bottlenecks or insufficient query processing capacity.\n\nIf it's consistently high while CPU usage remains low, consider increasing the concurrency limit (`-search.maxConcurrentRequests`) or optimizing query performance to support more concurrent users.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "links":[],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byName","options":"limit"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]}]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":236},
          "id":107,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},
          "pluginVersion":"12.3.0",
          "targets":[
            { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max(max_over_time(vt_concurrent_select_current{ins=~\"$ins\"}[$__rate_interval]))" ,"format":"time_series",             "interval":"","intervalFactor":1,"legendFormat":"current","range":true,"refId":"A" },
            { "datasource":{"type":"prometheus","uid":"$ds"}          ,"editorMode":"code","expr":"max(max_over_time(vt_concurrent_select_capacity{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"limit"  ,"range":true,"refId":"B" }
          ],
          "title":"Concurrent queries ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Number of queries that timed out while waiting for available concurrency slots. This indicates sustained query overload beyond configured limits.\n\nHigh values suggest:\n- Query queue is consistently full\n- Queries waiting too long for execution slots\n- System under sustained heavy load\n- Need for horizontal scaling or query optimization\n\nCombined with `Select concurrency limit reached`, provides complete picture of query rejection patterns.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"vis":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":1}]},
              "unit":"reqps"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":236},
          "id":132,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vt_concurrent_select_limit_timeout_total{ins=~\"$ins\"}[$__rate_interval])) by (instance) > 0","interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Query timeouts ($ins)",
          "type":"timeseries"
        }
      ],
      "title":"Querying",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":50},
      "id":126,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Number of storage blocks scanned per query (99th percentile). Each block contains trace spans for a specific time period and field combination. High values indicate queries scanning too many blocks, often caused by:\n\n- Wide time ranges without specific filters\n- Queries missing indexed fields (like `_stream`, `kubernetes.*`)\n- Non-selective filters that don't utilize `bloom filters`\n\nCorrelate with `Bytes/query p99` - if blocks are high but bytes are low, blocks contain little data (good). If both are high, query is reading large amounts of data.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":245},
          "id":130,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","exemplar":true,"expr":"max(histogram_quantile(0.99, sum(increase(vt_storage_per_query_processed_blocks_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))) by (instance)","hide":false,"interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Blocks/query p99 ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Total bytes read from disk per query (99th percentile). This represents the complete `I/O` overhead for query execution, including:\n\n- Block headers and metadata\n- Bloom filter data for candidate selection\n- Column headers and indexes\n- Actual trace span contents\n\nHigh values indicate expensive queries. Compare with specific breakdown panels below to identify bottlenecks. Monitor trends over time and correlate with query complexity.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":245},
          "id":121,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","exemplar":true,"expr":"max(histogram_quantile(0.99, sum(increase(vt_storage_per_query_total_read_bytes_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))) by (instance)","hide":false,"interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Bytes/query p99 ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Bytes read from block headers per query (99th percentile). Block headers contain metadata about each storage block including `time ranges`, `field names`, and data location pointers.\n\nHigh values indicate:\n- Query `time range` spans many blocks (reduce time range or add time-based filters)\n- Missing stream-level filters (`_stream` field) causing full block header scans\n- High cardinality fields creating excessive blocks\n\nMonitor relative changes over time - sudden increases suggest inefficient query patterns or changes in data structure.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":253},
          "id":124,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","exemplar":true,"expr":"max(histogram_quantile(0.99, sum(increase(vt_storage_per_query_block_headers_read_bytes_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))) by (instance) > 0.001","hide":false,"interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Block header bytes/query p99 ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Bytes read from Bloom filters per query (99th percentile). `Bloom filters` are probabilistic data structures that quickly eliminate blocks that definitely don't contain search terms.\n\nHigh values indicate:\n- Queries with low-selectivity text filters (common words like `error`, `info`)\n- Missing or ineffective field-based filters\n- Queries that force scanning many candidate blocks\n\nOptimize by adding specific field filters (`kubernetes.container_name`, `_stream`) before text searches. Monitor for sudden increases that indicate poor filter selectivity.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":253},
          "id":129,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","exemplar":true,"expr":"max(histogram_quantile(0.99, sum(increase(vt_storage_per_query_bloom_filters_read_bytes_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))) by (instance) > 0.001","hide":false,"interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Bloom filter bytes/query p99 ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Bytes read from actual trace span contents per query (99th percentile). This represents the uncompressed trace spans being retrieved and processed for the query result.\n\nHigh values indicate:\n- Queries returning large result sets (add `LIMIT` clause)\n- Retrieving trace spans with large payloads (`JSON` objects, stack traces)\n- Missing filters that would reduce matching trace span volume\n- Functions like `uniq` or `stats` processing many trace spans\n\nReduce by: adding selective filters, limiting result count.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":261},
          "id":122,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","exemplar":true,"expr":"max(histogram_quantile(0.99, sum(increase(vt_storage_per_query_values_read_bytes_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))) by (instance)","hide":false,"interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Value bytes/query p99 ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Bytes read from timestamp column (`_time`) per query (99th percentile). The `_time` column is automatically indexed and used for time-range filtering during query execution.\n\nHigh values indicate:\n- Queries with very wide time ranges requiring timestamp scanning\n- Time-based aggregations over large datasets\n- Missing time-range restrictions in query filters\n\nThis is usually the smallest component of query `I/O`. Spikes correlate with time range width and data density in the queried period.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":261},
          "id":127,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","exemplar":true,"expr":"max(histogram_quantile(0.99, sum(increase(vt_storage_per_query_timestamps_read_bytes_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))) by (instance)","hide":false,"interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"_time bytes/query p99 ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Bytes read from column header indexes per query (99th percentile). `Column header indexes` contain metadata about which fields exist in each block and their data types.\n\nHigh values suggest:\n- Queries scanning many blocks due to missing field filters\n- High field cardinality creating large index structures\n- Queries accessing many different field names across blocks\n- Schema evolution causing index fragmentation\n\nOptimize by using consistent `field names` and adding field-specific filters early in query pipeline.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":0,"y":269},
          "id":128,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","exemplar":true,"expr":"max(histogram_quantile(0.99, sum(increase(vt_storage_per_query_columns_header_indexes_read_bytes_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))) by (instance)","hide":false,"interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Column header index bytes/query p99 ($ins)",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Bytes read from column headers per query (99th percentile). `Column headers` store the actual field schema and compression metadata for each block's columns.\n\nThis metric helps identify:\n- Schema complexity overhead (many fields per trace span)\n- Inefficient field access patterns\n- Blocks with heterogeneous schemas requiring header reads\n\nCompare with other `I/O` breakdown panels to understand query cost distribution. High column header reads suggest schema optimization opportunities or need for more selective field access patterns.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic-by-name"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "min":0,
              "noValue":"0",
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"bytes"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":12,"x":12,"y":269},
          "id":123,
          "options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","exemplar":true,"expr":"max(histogram_quantile(0.99, sum(increase(vt_storage_per_query_columns_headers_read_bytes_bucket{ins=~\"$ins\"}[$__rate_interval])) by (instance, vmrange))) by (instance)","hide":false,"interval":"","legendFormat":"{{instance}}","range":true,"refId":"A"}],
          "title":"Column header bytes/query p99 ($ins)",
          "type":"timeseries"
        }
      ],
      "title":"Slow Query Troubleshooting",
      "type":"row"
    }
  ],
  "preload":false,
  "refresh":"",
  "schemaVersion":42,
  "tags":["Pigsty","INFRA","VICTORIA","VTRACES","INSTANCE"],
  "templating":{
    "list":[
      {"current":{},"hide":2,"includeAll":false,"name":"ds","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},
      {"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{cls=\"vtraces\"},ins)","includeAll":false,"name":"ins","options":[],"query":{"qryType":1,"query":"label_values(vm_app_version{cls=\"vtraces\"},ins)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"},
      {"current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{ins=\"$ins\"},ip)","hide":2,"name":"ip","options":[],"query":{"qryType":1,"query":"label_values(vm_app_version{ins=\"$ins\"},ip)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"},
      {"baseFilters":[],"datasource":{"type":"prometheus","uid":"$ds"},"filters":[],"name":"adhoc","type":"adhoc"}
    ]
  },
  "time":{"from":"now-1h","to":"now"},
  "timepicker":{},
  "timezone":"",
  "title":"Victoria Traces",
  "uid":"vtraces-instance",
  "version":1,
  "weekStart":""
}