{
  "annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},
  "author":"Ruohang Feng (rh@vonng.com)",
  "description":"Node key metrics and alerts",
  "editable":true,
  "fiscalYearStartMonth":0,
  "graphTooltip":0,
  "id":null,
  "license":"https://pigsty.io/docs/about/license/",
  "links":[{"asDropdown":true,"icon":"external link","includeVars":true,"keepTime":true,"tags":["Pigsty","NODE"],"targetBlank":false,"title":"NODE","tooltip":"","type":"dashboards","url":""}],
  "panels":[
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":16,"panels":[],"title":"Overview","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"decimals":0,"links":[],"mappings":[],"max":1,"min":0,"noValue":"OK","thresholds":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":4},{"color":"#b783af","value":8},{"color":"text","value":16}]},"unit":"none"},
        "overrides":[{"matcher":{"id":"byFrameRefID","options":"A"},"properties":[{"id":"links","value":[{"targetBlank":true,"title":"Pigsty Alertmanager","url":"/alertmgr/"}]}]},{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"links","value":[{"title":"PGSQL Alert","url":"/d/pgsql-alert?${__url_time_range}"}]}]}]
      },
      "gridPos":{"h":5,"w":2,"x":0,"y":1},
      "id":87,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(ALERTS{alertstate=\"firing\", job=\"node\"}) or on() vector(0)",            "interval":"","legendFormat":"Firing Alert" ,"range":true,"refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(ALERTS{alertstate=\"pending\", job=\"node\"})"                 ,"hide":true,"interval":"","legendFormat":"Pending Alert","range":true,"refId":"B" }
      ],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"thresholds"},
          "decimals":0,
          "links":[{"targetBlank":true,"title":"AlertInfo ${__field.labels.alertname}","url":"/alertmgr/#/alerts?filter=%7Balertname%3D%22${__field.labels.alertname}%22%7D"}],
          "mappings":[],
          "max":1,
          "min":0,
          "noValue":"OK",
          "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":2},{"color":"#b783af","value":4}]},
          "unit":"none"
        },
        "overrides":[]
      },
      "gridPos":{"h":5,"w":4,"x":2,"y":1},
      "id":86,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(ALERTS{alertstate=\"firing\", job=\"node\"}) by (alertname, severity)","interval":"","legendFormat":"{{ alertname }}","range":true,"refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"hidden",
            "axisSoftMax":1.2,
            "axisSoftMin":0,
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":98,
            "gradientMode":"none",
            "hideFrom":{"graph":false,"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"stepBefore",
            "lineWidth":0,
            "pointSize":1,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"normal"},
            "thresholdsStyle":{"mode":"off"}
          },
          "decimals":0,
          "links":[
            { "targetBlank":true,"title":"AlertInfo ${__field.labels.alertname} on Cluster ${__field.labels.cls}","url":"/alertmgr/#/alerts?filter=%7Bcls%3D%22${__field.labels.cls}%22%2C%20alertname%3D%22${__field.labels.alertname}%22%7D"       },
            { "targetBlank":true,"title":"Silence ${__field.labels.alertname} on Cluster ${__field.labels.cls}"  ,"url":"/alertmgr/#/silences/new?filter=%7Bcls%3D%22${__field.labels.cls}%22%2C%20alertname%3D%22${__field.labels.alertname}%22%7D" },
            {                    "title":"Node Cluster for ${__field.labels.cls} : Alerts"                       ,"url":"/d/node-cluster?var-cls=${__field.labels.cls}&viewPanel=227&${__url_time_range}"                                            }
          ],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"none"
        },
        "overrides":[{"matcher":{"id":"byFrameRefID","options":"A"},"properties":[{"id":"custom.fillOpacity","value":66}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"auto"}]}]
      },
      "gridPos":{"h":5,"w":18,"x":6,"y":1},
      "id":34,
      "options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum without (ins,ip,instance,datname) (ALERTS{alertstate=\"firing\", job=\"node\"})",            "interval":"","legendFormat":"[{{ severity }}] {{alertname}} @ {{ cls }}","range":true,"refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},                    "exemplar":false,"expr":"0"                                                                                  ,"hide":true,"interval":"","legendFormat":"Alert"                                     ,             "refId":"C" }
      ],
      "title":"Firing Alerts",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"custom":{"align":"center","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"decimals":0,"links":[],"mappings":[],"max":-4,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#e3e3e3e0","value":0}]},"unit":"none"},
        "overrides":[
          {"matcher":{"id":"byName","options":"Alert"},"properties":[{"id":"custom.width"},{"id":"links","value":[{"targetBlank":true,"title":"AlertInfo for ${__field.labels.alertname} on Cluster ${__field.labels.cls}","url":"/alertmgr/#/alerts?filter=%7Bcls%3D%22${__data.fields.Cluster}%22%2C%20alertname%3D%22${__data.fields.Alert}%22%7D"}]}]},
          {"matcher":{"id":"byName","options":"State"},"properties":[{"id":"custom.width","value":80},{"id":"mappings","value":[{"options":{"firing":{"color":"light-red","index":0,"text":"Firing\ud83d\udd25"},"pending":{"color":"light-orange","index":1,"text":"Pend\u23f0"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"mode":"gradient","type":"color-background"}}]},
          {"matcher":{"id":"byName","options":"Level"},"properties":[{"id":"custom.width","value":60},{"id":"mappings","value":[{"options":{"0":{"color":"red","index":0,"text":"P0"},"1":{"color":"#f79f64","index":1,"text":"P1"},"2":{"color":"#fcdb72","index":2,"text":"P2"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"mode":"gradient","type":"color-background"}}]},
          {
            "matcher":{"id":"byName","options":"Severity"},
            "properties":[{"id":"custom.width","value":110},{"id":"mappings","value":[{"options":{"CRIT":{"color":"red","index":0,"text":"Critical \u26a1\ufe0f"},"INFO":{"color":"light-yellow","index":2,"text":"Info \u2139\ufe0f"},"WARN":{"color":"super-light-orange","index":1,"text":"Warning \u26a0\ufe0f"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"mode":"gradient","type":"color-background"}}]
          },
          {"matcher":{"id":"byName","options":"Category"},"properties":[{"id":"custom.width","value":80},{"id":"mappings","value":[{"options":{"infra":{"color":"#f79f64","index":2,"text":"INFRA"},"node":{"color":"#346f36cc","index":1,"text":"NODE"},"pgsql":{"color":"light-blue","index":0,"text":"PGSQL"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"mode":"gradient","type":"color-background"}}]},
          {"matcher":{"id":"byName","options":"Ins"},"properties":[{"id":"custom.width","value":100}]},{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"custom.width","value":66}]},
          {"matcher":{"id":"byName","options":"Cluster"},"properties":[{"id":"custom.width"},{"id":"links","value":[{"title":"PGSQL Cluster for ${cls}","url":"/d/pgsql-cluster?var-cls=${__data.fields.Cluster}&${__url_time_range}"}]}]},
          {"matcher":{"id":"byName","options":"Instance"},"properties":[{"id":"links","value":[{"title":"PGSQL Instance for ${__data.fields.Instance}","url":"/d/pgsql-instance?var-ins=${__data.fields.Instance}&${__url_time_range}"}]}]},
          {"matcher":{"id":"byName","options":"Database"},"properties":[{"id":"custom.width","value":80},{"id":"links","value":[{"title":"PGSQL Database for ${__data.fields.Database} on ${__data.fields.Instance}","url":"/d/pgsql-database?var-ins=${__data.fields.Instance}&var-datname=${__data.fields.Database}&${__url_time_range}"}]}]},
          {
            "matcher":{"id":"byName","options":"Silence"},
            "properties":[{"id":"links","value":[{"targetBlank":true,"title":"Silence ${__data.fields.Alert} on Cluster ${__data.fields.Cluster}","url":"/alertmgr/#/silences/new?filter=%7Bcls%3D%22${__data.fields.Cluster}%22%2C%20alertname%3D%22${__data.fields.Alert}%22%7D"}]},{"id":"mappings","value":[{"options":{"1":{"index":0,"text":"silence"}},"type":"value"}]},{"id":"custom.width","value":70}]
          },
          {"matcher":{"id":"byName","options":"Agent"},"properties":[{"id":"custom.width","value":140}]},{"matcher":{"id":"byName","options":"IP"},"properties":[{"id":"custom.width","value":110},{"id":"links","value":[{"title":"PGSQL Node: ${__data.fields.IP}","url":"/d/node-instance?var-id=${__data.fields.IP}&${__url_time_range}"}]}]}
        ]
      },
      "gridPos":{"h":8,"w":24,"x":0,"y":6},
      "id":36,
      "options":{"cellHeight":"sm","showHeader":true,"sortBy":[{"desc":false,"displayName":"Level"}]},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"ALERTS{job=\"node\"}","format":"table","instant":true,"interval":"","legendFormat":"","refId":"A"}],
      "title":"Active Alerts",
      "transformations":[
        {
          "id":"organize",
          "options":{
            "excludeByName":{"Time":true,"Value":false,"__name__":true,"job":true},
            "indexByName":{"Time":11,"Value":10,"__name__":13,"alertname":0,"alertstate":1,"category":4,"cls":5,"datname":8,"ins":6,"instance":9,"ip":7,"job":12,"level":2,"severity":3},
            "renameByName":{"Value":"Silence","Value #A":"Silence","alertname":"Alert","alertstate":"State","category":"Category","cls":"Cluster","datname":"Database","ins":"Instance","instance":"Agent","ip":"IP","job":"","level":"Level","severity":"Severity"}
          }
        }
      ],
      "type":"table"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "axisSoftMin":0,
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"auto",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"area"}
          },
          "decimals":1,
          "links":[{"title":"PGSQL Node : CPU for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=172&${__url_time_range}"}],
          "mappings":[],
          "max":1,
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"green","value":0.1},{"color":"yellow","value":0.3},{"color":"orange","value":0.5},{"color":"red","value":0.7},{"color":"purple","value":0.9},{"color":"text","value":1}]},
          "unit":"percentunit"
        },
        "overrides":[]
      },
      "gridPos":{"h":8,"w":24,"x":0,"y":14},
      "id":18,
      "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:cpu_usage_1m","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
      "title":"NodeCpuHigh",
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":82,"panels":[],"title":"Error Logs","type":"row"},
    {
      "datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
      "description":"Filter syslog with priority < 4, emerg, alert, crit, error",
      "gridPos":{"h":13,"w":24,"x":0,"y":23},
      "id":84,
      "maxDataPoints":100,
      "options":{"dedupStrategy":"none","detailsMode":"sidebar","enableInfiniteScrolling":true,"enableLogDetails":true,"fontSize":"small","prettifyLogMessage":false,"showCommonLabels":false,"showControls":true,"showLabels":true,"showTime":true,"sortOrder":"Descending","wrapLogMessage":false},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog p:<4 | fields level,unit,ins,ip,_msg","queryType":"instant","refId":"A"}],
      "title":"Node Syslog Error",
      "type":"logs"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":36},
      "id":53,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Standard Load (divide by cpu count) > 100%",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMax":1,
                "axisSoftMin":0,
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "decimals":1,
              "links":[{"title":"PGSQL Node: Load for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=173&${__url_time_range}"}],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.1},{"color":"yellow","value":0.3},{"color":"orange","value":0.5},{"color":"red","value":0.7},{"color":"purple","value":0.9},{"color":"text","value":1}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":24,"x":0,"y":55},
          "id":11,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:stdload1","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
          "title":"NodeLoadHigh",
          "type":"timeseries"
        }
      ],
      "title":"Node: Schedule",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":37},
      "id":85,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Available memory < 10%",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMin":0,
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "decimals":1,
              "links":[{"title":"PGSQL Node: Memory for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=40&${__url_time_range}"}],
              "mappings":[],
              "max":1,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"purple"},{"color":"red","value":0.1},{"color":"orange","value":0.15},{"color":"green","value":0.2}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":24,"x":0,"y":64},
          "id":56,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"node:ins:mem_avail","interval":"","legendFormat":"{{ ip }}","range":true,"refId":"A"}],
          "title":"NodeOutOfMem",
          "type":"timeseries"
        }
      ],
      "title":"Node: Memory",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":38},
      "id":55,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Available memory < 10%",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMin":0,
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "links":[{"title":"PGSQL Node: Swap for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=33&${__url_time_range}"}],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":0.01},{"color":"red","value":0.05},{"color":"purple","value":0.1}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":24,"x":0,"y":73},
          "id":57,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:swap_usage","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
          "title":"NodeMemSwapped",
          "type":"timeseries"
        }
      ],
      "title":"Node: Swap",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":39},
      "id":27,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Disk read/write avg latency > 32ms",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMin":0,
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "links":[{"title":"PGSQL Node: Disk Latency for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=29&${__url_time_range}"}],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.001},{"color":"orange","value":0.008},{"color":"red","value":0.032},{"color":"purple","value":0.128},{"color":"text","value":1}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":24,"x":0,"y":82},
          "id":60,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"max by (ip,device) (node:dev:disk_write_rt_1m > node:dev:disk_read_rt_1m or node:dev:disk_read_rt_1m)","interval":"","legendFormat":"{{ ip}}-{{ device }}","refId":"A"}],
          "title":"NodeDiskSlow",
          "type":"timeseries"
        }
      ],
      "title":"NODE: Disk",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":40},
      "id":29,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMax":1,
                "axisSoftMin":0,
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "links":[{"title":"PGSQL Node: Space Usage for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=199&${__url_time_range}"}],
              "mappings":[],
              "max":3,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.1},{"color":"yellow","value":0.8},{"color":"orange","value":0.9},{"color":"red","value":0.95},{"color":"purple","value":0.99},{"color":"text","value":1}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":8,"w":24,"x":0,"y":91},
          "id":23,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:fs:space_usage","interval":"","legendFormat":"{{ ip }}:{{ mountpoint }}","refId":"A"}],
          "title":"NodeFsFull",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"opacity",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "spanNulls":true,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "decimals":0,
              "links":[{"title":"PGSQL Node: Space Exhaust for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=51&${__url_time_range}"}],
              "mappings":[{"options":{"-1":{"index":1,"text":"+Inf"}},"type":"value"},{"options":{"match":"null+nan","result":{"index":0,"text":"N/A"}},"type":"special"}],
              "max":3200000,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"text"},{"color":"purple","value":3600},{"color":"orange","value":86400},{"color":"yellow","value":604800},{"color":"#346f36cc","value":126230400}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":7,"w":24,"x":0,"y":99},
          "id":78,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "pluginVersion":"8.0.0",
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:fs:space_exhaust{}","interval":"","legendFormat":"{{ ip }}:{{ mountpoint }}","refId":"A"}],
          "title":"NodeFsExhaust",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMax":1,
                "axisSoftMin":0,
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "links":[{"title":"PGSQL Node: iNode Usage for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=174&${__url_time_range}"}],
              "mappings":[],
              "max":3,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.1},{"color":"yellow","value":0.8},{"color":"orange","value":0.9},{"color":"red","value":0.95},{"color":"purple","value":0.99},{"color":"text","value":1}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":7,"w":24,"x":0,"y":106},
          "id":58,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "targets":[{"exemplar":false,"expr":"node:fs:inode_usage","interval":"","legendFormat":"{{ ins }}:{{ mountpoint }}","refId":"A"}],
          "title":"NodeFsFilesFull",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMax":1,
                "axisSoftMin":0,
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "links":[{"title":"PGSQL Node: FD for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=58&${__url_time_range}"}],
              "mappings":[],
              "max":3,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.1},{"color":"orange","value":0.8},{"color":"orange","value":0.9},{"color":"red","value":0.95},{"color":"purple","value":0.99},{"color":"text","value":1}]},
              "unit":"percentunit"
            },
            "overrides":[]
          },
          "gridPos":{"h":7,"w":24,"x":0,"y":113},
          "id":59,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:fd_usage","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
          "title":"NodeFdFull",
          "type":"timeseries"
        }
      ],
      "title":"NODE: Filesystem",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":41},
      "id":62,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"TCP Error Rate > 1 / s",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMin":0,
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "decimals":1,
              "links":[{"title":"PGSQL Node: TCP Errors for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=55&${__url_time_range}"}],
              "mappings":[],
              "max":1,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":1},{"color":"red","value":2}]},
              "unit":"cps"
            },
            "overrides":[]
          },
          "gridPos":{"h":7,"w":24,"x":0,"y":121},
          "id":71,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"rate(node:ins:tcp_error[1m])","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
          "title":"NodeTcpErrHigh",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "axisSoftMax":1,
                "axisSoftMin":0,
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"log":10,"type":"log"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "links":[{"title":"PGSQL Node: TCP Retransmit for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=52&${__url_time_range}"}],
              "mappings":[],
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":1e-06},{"color":"yellow","value":0.0001},{"color":"orange","value":0.001},{"color":"red","value":0.01},{"color":"purple","value":0.1}]},
              "unit":"sci"
            },
            "overrides":[]
          },
          "gridPos":{"h":9,"w":24,"x":0,"y":128},
          "id":70,
          "options":{"legend":{"calcs":["max"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:tcp_retrans_ratio1m","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
          "title":"NodeTcpRetransHigh",
          "type":"timeseries"
        }
      ],
      "title":"Node: Network",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":42},
      "id":25,
      "panels":[
        {
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Check ntp protocol / clock sanity",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"hidden",
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "links":[{"title":"PGSQL Node: NTP Sanity for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=70&${__url_time_range}"}],
              "mappings":[],
              "max":1.1,
              "min":0,
              "thresholds":{"mode":"absolute","steps":[{"color":"red"},{"color":"green","value":0.5}]},
              "unit":"none"
            },
            "overrides":[]
          },
          "gridPos":{"h":7,"w":24,"x":0,"y":51},
          "id":75,
          "options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Min","sortDesc":false},"tooltip":{"mode":"single","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min without() (node_timex_sync_status)","interval":"","legendFormat":"{{ ip }}","range":true,"refId":"A"}],
          "title":"NodeTimeSync",
          "type":"timeseries"
        },{
          "datasource":{"type":"prometheus","uid":"ds-prometheus"},
          "description":"Clock time offset alert > 64ms",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"log":2,"type":"log"},
                "showPoints":"auto",
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"area"}
              },
              "links":[{"title":"PGSQL Node: Time Offset for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=83&${__url_time_range}"}],
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.001},{"color":"yellow","value":0.01},{"color":"orange","value":0.064},{"color":"red","value":1},{"color":"purple","value":60},{"color":"text","value":600}]},
              "unit":"s"
            },
            "overrides":[]
          },
          "gridPos":{"h":7,"w":24,"x":0,"y":58},
          "id":74,
          "options":{"legend":{"calcs":["max"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},
          "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"abs(node_timex_offset_seconds)","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
          "title":"NodeClockSkew",
          "type":"timeseries"
        }
      ],
      "title":"Node: Time",
      "type":"row"
    }
  ],
  "preload":false,
  "refresh":"",
  "schemaVersion":42,
  "tags":["Pigsty","NODE","Overview","Alert"],
  "templating":{"list":[]},
  "time":{"from":"now-1h","to":"now"},
  "timepicker":{"refresh_intervals":["2s","5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"]},
  "timezone":"browser",
  "title":"Node Alert",
  "uid":"node-alert",
  "version":1,
  "weekStart":""
}