{
  "annotations":{
    "list":[
      {"builtIn":1,"datasource":{"uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"},
      {"datasource":{"uid":"$datasource"},"enable":true,"expr":"changes(process_start_time_seconds{ ins=~\"$ins\"}[2m]) > 0","hide":false,"iconColor":"#bf1b00","name":"Restarts","showIn":0,"step":"1m","tagKeys":"instance","titleFormat":"Restart"}
    ]
  },
  "author":"Ruohang Feng (rh@vonng.com)",
  "description":"Dashboard showing Prometheus Alertmanager metrics for observing status of the cluster and possible debbuging.",
  "editable":true,
  "fiscalYearStartMonth":0,
  "graphTooltip":1,
  "id":null,
  "license":"https://pigsty.io/docs/about/license/",
  "links":[{"asDropdown":true,"icon":"external link","includeVars":false,"keepTime":true,"tags":["Pigsty","INFRA"],"targetBlank":false,"title":"INFRA","tooltip":"","type":"dashboards","url":""}],
  "panels":[
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":200,"panels":[],"title":"Overview","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]}},"overrides":[{"matcher":{"id":"byName","options":"IP"},"properties":[{"id":"links","value":[{"title":"Node Instance : $ip","url":"/d/node-instance?var-id=${ip}&${__url_time_range}"}]}]}]},
      "gridPos":{"h":6,"w":5,"x":0,"y":1},
      "id":201,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"/.*/","values":true},"showPercentChange":false,"text":{"titleSize":16,"valueSize":16},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"alertmanager_build_info{ins=\"$ins\"}","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],
      "transformations":[
        {
          "id":"organize",
          "options":{
            "excludeByName":{"Time":true,"Value":true,"__name__":true,"branch":true,"cls":true,"goarch":true,"goos":true,"goversion":true,"job":true,"revision":true,"short_version":false,"tags":true,"version":false},
            "includeByName":{},
            "indexByName":{"Time":5,"Value":8,"__name__":6,"branch":9,"cls":3,"goarch":10,"goos":11,"goversion":12,"ins":0,"instance":1,"ip":2,"job":7,"revision":13,"tags":14,"version":4},
            "orderByMode":"manual",
            "renameByName":{"Value":"","cls":"Cluster","ins":"Instance","instance":"Address","ip":"IP","job":"","short_version":"Version","version":"Version"}
          }
        }
      ],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"All AlertManager instances",
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"custom":{"align":"center","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#e3e3e3e0","value":0}]}},
        "overrides":[
          { "matcher":{"id":"byName","options":"Instance"},"properties":[{"id":"links","value":[{"title":"AlertManager Instance ${__data.fields.Instance}","url":"/d/alertmanager-instance?var-ins=${__data.fields.Instance}&${__url_time_range}"}]},{"id":"custom.width"}]                                                                                                                                   },
          { "matcher":{"id":"byName","options":"IP"}      ,"properties":[{"id":"custom.width","value":110},{"id":"links","value":[{"title":"Node Instance : ${__data.fields.IP}","url":"/d/node-instance?var-id=${__data.fields.IP}&${__url_time_range}"}]}]                                                                                                                                                  },
          { "matcher":{"id":"byName","options":"Up"}      ,"properties":[{"id":"custom.width","value":50},{"id":"mappings","value":[{"options":{"0":{"color":"#cc4637d9","index":0,"text":"Dead"},"1":{"color":"#346f36cc","index":1,"text":"Alive"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"type":"color-background"}}]                                                                      },
          { "matcher":{"id":"byName","options":"Alerts"}  ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":10},{"color":"red","value":50}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                          },
          { "matcher":{"id":"byName","options":"Silenced"},"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":1}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                                                                                    },
          { "matcher":{"id":"byName","options":"Notif/s"} ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"short"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":0.01},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":10}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}]                    },
          { "matcher":{"id":"byName","options":"Memory"}  ,"properties":[{"id":"custom.width","value":80},{"id":"unit","value":"bytes"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#346f36cc","value":67108864},{"color":"#fcdb72","value":268435456},{"color":"#f79f64","value":536870912}]}},{"id":"custom.cellOptions","value":{"type":"color-text"}}] }
        ]
      },
      "gridPos":{"h":6,"w":15,"x":5,"y":1},
      "id":202,
      "options":{"cellHeight":"sm","showHeader":true,"sortBy":[{"desc":false,"displayName":"Instance"}]},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min by (ins,ip) (up{cls=\"alertmanager\"})"                                                   ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (alertmanager_alerts{cls=\"alertmanager\"})"                                     ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"B" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (alertmanager_silences{cls=\"alertmanager\", state=\"active\"})"                 ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"C" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (rate(alertmanager_notifications_total{cls=\"alertmanager\"}[$__rate_interval]))","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"D" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum by (ins) (process_resident_memory_bytes{cls=\"alertmanager\"})"                           ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"E" }
      ],
      "transformations":[
        {"id":"seriesToColumns","options":{"byField":"ins"}},
        {
          "id":"organize",
          "options":{
            "excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true},
            "includeByName":{},
            "indexByName":{"Time 1":7,"Time 2":8,"Time 3":9,"Time 4":10,"Time 5":11,"Value #A":0,"Value #B":3,"Value #C":4,"Value #D":5,"Value #E":6,"ins":1,"ip":2},
            "renameByName":{"Value #A":"Up","Value #B":"Alerts","Value #C":"Silenced","Value #D":"Notif/s","Value #E":"Memory","ins":"Instance","ip":"IP"}
          }
        }
      ],
      "type":"table"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"fixedColor":"#db7070","mode":"fixed"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":30,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":true,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":4,"x":20,"y":1},
      "id":203,
      "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"sum(alertmanager_alerts{ins=~\"$ins\"})","format":"time_series","intervalFactor":1,"legendFormat":"Active Alerts","range":true,"refId":"A"}],
      "title":"Active Alerts",
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":7},"id":210,"panels":[],"title":"Key Metrics","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"How long the AlertManager instance has been running",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]},"unit":"dtdurations"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":0,"y":8},
      "id":211,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"(time() - process_start_time_seconds{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Uptime","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total number of active alerts",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":3,"y":8},
      "id":212,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(alertmanager_alerts{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Active Alerts","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Number of active silences",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":6,"y":8},
      "id":213,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(alertmanager_silences{ins=~\"$ins\", state=\"active\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Silences","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Rate of notifications sent",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]},"unit":"ops"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":9,"y":8},
      "id":214,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(alertmanager_notifications_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"Notifications","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Rate of failed notifications",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]},"unit":"ops"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":12,"y":8},
      "id":215,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(alertmanager_notifications_failed_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"Notif Failed","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Current resident memory usage of the process",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]},"unit":"bytes"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":15,"y":8},
      "id":216,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(process_resident_memory_bytes{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Memory","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Number of AlertManager cluster members",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]},"unit":"short"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":18,"y":8},
      "id":217,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(alertmanager_cluster_members{ins=~\"$ins\"})","format":"time_series","instant":true,"interval":"","legendFormat":"Cluster Size","refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Rate of cluster gossip messages",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"#db7070","value":0}]},"unit":"ops"},"overrides":[]},
      "gridPos":{"h":3,"w":3,"x":21,"y":8},
      "id":218,
      "options":{"colorMode":"background_solid","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(rate(alertmanager_cluster_messages_received_total{ins=~\"$ins\"}[$__rate_interval]))","format":"time_series","instant":true,"interval":"","legendFormat":"Gossip Msgs","refId":"A"}],
      "type":"stat"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":11},"id":315,"panels":[],"title":"Logging","type":"row"},
    {
      "datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"fixedColor":"#346f36cc","mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"bars",
            "fillOpacity":100,
            "gradientMode":"none",
            "hideFrom":{"graph":false,"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":true,
            "stacking":{"group":"A","mode":"normal"},
            "thresholdsStyle":{"mode":"off"}
          },
          "decimals":0,
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"none"
        },
        "overrides":[]
      },
      "gridPos":{"h":4,"w":24,"x":0,"y":12},
      "id":316,
      "interval":"1s",
      "options":{"legend":{"calcs":["sum"],"displayMode":"table","placement":"right","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog ip:$ip app:alertmanager | stats count() as count","legendFormat":"syslog","queryType":"statsRange","refId":"A"}],
      "type":"timeseries"
    },
    {
      "datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
      "description":"syslog of grafana-server in $ins @ $ip",
      "gridPos":{"h":10,"w":24,"x":0,"y":16},
      "id":317,
      "maxDataPoints":100,
      "options":{"dedupStrategy":"none","detailsMode":"sidebar","enableInfiniteScrolling":true,"enableLogDetails":true,"fontSize":"small","prettifyLogMessage":false,"showCommonLabels":true,"showControls":true,"showLabels":true,"showTime":true,"sortOrder":"Descending","syntaxHighlighting":true,"wrapLogMessage":false},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog ip:$ip unit:alertmanager","queryType":"range","refId":"A"}],
      "type":"logs"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":26},"id":113,"panels":[],"title":"Notifications","type":"row"},
    {
      "datasource":{"uid":"$datasource"},
      "description":"Number of sent notifications to distinct integrations such as PagerDuty, Slack and so on. On negative axis are displayed failed notifications.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[
          { "matcher":{"id":"byRegexp","options":"/Failed.*/"}                               ,"properties":[{"id":"color","value":{"fixedColor":"#99440a","mode":"fixed"}},{"id":"custom.transform","value":"negative-Y"}] },
          { "matcher":{"id":"byValue","options":{"op":"gte","reducer":"allIsZero","value":0}},"properties":[{"id":"custom.hideFrom","value":{"legend":true,"tooltip":true,"viz":false}}]                                   },
          { "matcher":{"id":"byValue","options":{"op":"gte","reducer":"allIsNull","value":0}},"properties":[{"id":"custom.hideFrom","value":{"legend":true,"tooltip":true,"viz":false}}]                                   }
        ]
      },
      "gridPos":{"h":6,"w":12,"x":0,"y":27},
      "id":118,
      "options":{"legend":{"calcs":["mean","sum"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "repeat":"instance",
      "repeatDirection":"h",
      "targets":[
        { "datasource":{"uid":"$datasource"},"expr":"sum(increase(alertmanager_notifications_total{ins=~\"$ins\"}[$__interval])) by (integration)"       ,"format":"time_series",             "intervalFactor":1,              "legendFormat":"{{ integration}}"        ,"refId":"B" },
        { "datasource":{"uid":"$datasource"},"expr":"sum(increase(alertmanager_notifications_failed_total{ins=~\"$ins\"}[$__interval])) by (integration)","format":"time_series","hide":false,"intervalFactor":1,"interval":"","legendFormat":"Failed {{ integration }}","refId":"A" }
      ],
      "title":"Notifications sent from $ins",
      "type":"timeseries"
    },
    {
      "datasource":{"uid":"$datasource"},
      "description":"Duration of notification sends in 0.99 and 0.9 quantiles per integration.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":true,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
          "unit":"s"
        },
        "overrides":[
          { "matcher":{"id":"byRegexp","options":"/0.99.*/"}                                 ,"properties":[{"id":"custom.lineWidth","value":1}]                                         },{ "matcher":{"id":"byRegexp","options":"/0.5 .*/"}                                 ,"properties":[{"id":"custom.lineWidth","value":2}]                                         },
          { "matcher":{"id":"byValue","options":{"op":"gte","reducer":"allIsZero","value":0}},"properties":[{"id":"custom.hideFrom","value":{"legend":true,"tooltip":true,"viz":false}}] },{ "matcher":{"id":"byValue","options":{"op":"gte","reducer":"allIsNull","value":0}},"properties":[{"id":"custom.hideFrom","value":{"legend":true,"tooltip":true,"viz":false}}] }
        ]
      },
      "gridPos":{"h":6,"w":12,"x":12,"y":27},
      "id":115,
      "options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "repeat":"instance",
      "repeatDirection":"h",
      "targets":[
        { "datasource":{"uid":"$datasource"},"expr":"sum(histogram_quantile(0.9,rate(alertmanager_notification_latency_seconds_bucket{ins=~\"$ins\"}[$__interval]))) by (integration)" ,"format":"time_series",              "intervalFactor":1,"legendFormat":"0.9q {{ integration }}" ,"refId":"B" },
        { "datasource":{"uid":"$datasource"},"expr":"sum(histogram_quantile(0.99,rate(alertmanager_notification_latency_seconds_bucket{ins=~\"$ins\"}[$__interval]))) by (integration)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"0.99q {{ integration }}","refId":"A" }
      ],
      "title":"Notification durations per integration on $ins",
      "type":"timeseries"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":33},
      "id":18,
      "panels":[
        {
          "datasource":{"uid":"$datasource"},
          "description":"Number of alerts by state such as `active`, `suppressed` etc.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":40,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"normal"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byName","options":"active"},"properties":[{"id":"color","value":{"fixedColor":"#bf1b00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"suppressed"},"properties":[{"id":"color","value":{"fixedColor":"#2f575e","mode":"fixed"}}]}]
          },
          "gridPos":{"h":6,"w":12,"x":0,"y":34},
          "id":6,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"sum(alertmanager_alerts{ins=~\"$ins\"}) by (state)","format":"time_series","intervalFactor":1,"legendFormat":"{{state}}","refId":"A"}],
          "title":"Active alerts in $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "description":"Number of received alerts from Prometheus by status `firing` on positive axis and `resolved` on negative axis.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":10,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"never",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},
              "unit":"short"
            },
            "overrides":[{"matcher":{"id":"byName","options":"resolved"},"properties":[{"id":"color","value":{"fixedColor":"#7eb26d","mode":"fixed"}},{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byName","options":"firing"},"properties":[{"id":"color","value":{"fixedColor":"#99440a","mode":"fixed"}}]}]
          },
          "gridPos":{"h":6,"w":12,"x":12,"y":34},
          "id":8,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"sum(increase(alertmanager_alerts_received_total{ins=~\"$ins\"}[$__interval])) by (status)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{ status }}","refId":"A"}],
          "title":"Received alerts by status for $ins",
          "type":"timeseries"
        }
      ],
      "title":"Alerts",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":34},
      "id":34,
      "panels":[
        {
          "datasource":{"uid":"$datasource"},
          "description":"Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":12,"x":0,"y":62},
          "id":57,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"alertmanager_cluster_health_score{ins=~\"$ins\"}","format":"time_series","intervalFactor":1,"legendFormat":"Cluster health score","refId":"A"}],
          "title":"Clusterhealth score for $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "description":"Shows gossip cluster members count in time and failing peers in case of any in red color.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":12,"x":12,"y":62},
          "id":38,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"alertmanager_cluster_members{ins=~\"$ins\"}","format":"time_series","intervalFactor":1,"legendFormat":"Number of cluster members","refId":"A"},{"datasource":{"uid":"$datasource"},"expr":"alertmanager_cluster_failed_peers{ins=~\"$ins\"}","format":"time_series","intervalFactor":1,"legendFormat":"Number of failed peers","refId":"B"}],
          "title":"Cluster members count on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "description":"On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":12,"x":0,"y":68},
          "id":75,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_cluster_peers_left_total{ins=~\"$ins\"}[$__interval])"  ,"format":"time_series","intervalFactor":1,"legendFormat":"Cluster joined peers","refId":"A" },
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_cluster_peers_joined_total{ins=~\"$ins\"}[$__interval])","format":"time_series","intervalFactor":1,"legendFormat":"Cluster left peers"  ,"refId":"B" }
          ],
          "title":"Cluster peers left/joined on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "description":"On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":12,"x":12,"y":68},
          "id":68,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_cluster_reconnections_total{ins=~\"$ins\"}[$__interval])"       ,"format":"time_series","intervalFactor":1,"legendFormat":"Sucessful reconnections","refId":"A" },
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_cluster_reconnections_failed_total{ins=~\"$ins\"}[$__interval])","format":"time_series","intervalFactor":1,"legendFormat":"Failed reconnections"   ,"refId":"B" }
          ],
          "title":"Cluster reconnections on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "description":"On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":12,"x":0,"y":74},
          "id":48,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            { "datasource":{"uid":"$datasource"},"expr":"sum(increase(alertmanager_cluster_messages_sent_total{ins=~\"$ins\"}[$__interval])) by (msg_type)"    ,"format":"time_series","intervalFactor":1,"legendFormat":"sent {{msg_type}}"    ,"refId":"A" },
            { "datasource":{"uid":"$datasource"},"expr":"sum(increase(alertmanager_cluster_messages_received_total{ins=~\"$ins\"}[$__interval])) by (msg_type)","format":"time_series","intervalFactor":1,"legendFormat":"received {{msg_type}}","refId":"B" }
          ],
          "title":"Cluster messages count on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "description":"On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":12,"x":12,"y":74},
          "id":53,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            { "datasource":{"uid":"$datasource"},"expr":"sum(increase(alertmanager_cluster_messages_sent_size_total{ins=~\"$ins\"}[$__interval])) by (msg_type)"    ,"format":"time_series","intervalFactor":1,"legendFormat":"sent {{msg_type}}"    ,"refId":"A" },
            { "datasource":{"uid":"$datasource"},"expr":"sum(increase(alertmanager_cluster_messages_received_size_total{ins=~\"$ins\"}[$__interval])) by (msg_type)","format":"time_series","intervalFactor":1,"legendFormat":"received {{msg_type}}","refId":"B" }
          ],
          "title":"Cluster messages size on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "description":"On positive axis is number of queued cluster messages and on negative axis number of pruned messages.",
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":4,"w":24,"x":0,"y":80},
          "id":62,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"alertmanager_cluster_messages_pruned_total{ins=~\"$ins\"}","format":"time_series","intervalFactor":1,"legendFormat":"Pruned messaged","refId":"A"},{"datasource":{"uid":"$datasource"},"expr":"alertmanager_cluster_messages_queued{ins=~\"$ins\"}","format":"time_series","intervalFactor":1,"legendFormat":"Queued messages","refId":"B"}],
          "title":"Cluster messages queue on $ins",
          "type":"timeseries"
        }
      ],
      "title":"Cluster members",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":35},
      "id":284,
      "panels":[
        {
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":85},
          "id":314,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_oversized_gossip_message_sent_total{ins=~\"$ins\"}[$__interval])"   ,"format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{key}}"        ,"refId":"A" },
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_oversized_gossip_message_dropped_total{ins=~\"$ins\"}[$__interval])","format":"time_series",             "intervalFactor":1,"legendFormat":"dropped {{key}}","refId":"B" },
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_oversized_gossip_message_failure_total{ins=~\"$ins\"}[$__interval])","format":"time_series",             "intervalFactor":1,"legendFormat":"failed {{key}}" ,"refId":"C" }
          ],
          "title":"Count of oversized gossip messages on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":91},
          "id":307,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{ins=~\"$ins\"}[$__interval]))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{key}}","refId":"A"}],
          "title":"Duration of oversized gossip messages on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":97},
          "id":303,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_silences_gossip_messages_propagated_total{ins=~\"$ins\"}[$__interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"silences","refId":"A" },
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_nflog_gossip_messages_propagated_total{ins=~\"$ins\"}[$__interval])"   ,"format":"time_series",             "intervalFactor":1,"legendFormat":"nf_log"  ,"refId":"B" }
          ],
          "title":"Number of propagated gossip messages on $ins",
          "type":"timeseries"
        }
      ],
      "title":"Gossip messages",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":36},
      "id":84,
      "panels":[
        {
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":104},
          "id":94,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_nflog_queries_total{ins=~\"$ins\"}[$__interval])"     ,"format":"time_series","intervalFactor":1,"legendFormat":"Nf log query count" ,"refId":"A" },
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_nflog_query_errors_total{ins=~\"$ins\"}[$__interval])","format":"time_series","intervalFactor":1,"legendFormat":"Nf log query errors","refId":"B" }
          ],
          "title":"Nf log queries count for $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":110},
          "id":106,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{ins=~\"$ins\"}[$__interval]))","format":"time_series","intervalFactor":1,"legendFormat":"Nf log query duration","refId":"A"}],
          "title":"Nf log query duration for $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":116},
          "id":97,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"alertmanager_nflog_snapshot_size_bytes{ins=~\"$ins\"}","format":"time_series","intervalFactor":1,"legendFormat":"Nf log snapshot size","refId":"A"}],
          "title":"Nf log snapshot size for $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":122},
          "id":101,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"rate(alertmanager_nflog_snapshot_duration_seconds_sum{ins=~\"$ins\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{ins=~\"$ins\"}[$__interval])","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Nf log snapshot size","refId":"A"}],
          "title":"Nf log snapshot duration for $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":128},
          "id":92,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"alertmanager_nflog_gc_duration_seconds{ins=~\"$ins\"}","format":"time_series","intervalFactor":1,"legendFormat":"Cluster joined peers","refId":"A"}],
          "title":"Nf log Go GC time for $ins",
          "type":"timeseries"
        }
      ],
      "title":"Nflog",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":37},
      "id":123,
      "panels":[
        {
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":135},
          "id":129,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"alertmanager_silences{ins=~\"$ins\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"{{state}}","refId":"A"}],
          "title":"Silences count by state on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":141},
          "id":134,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_silences_queries_total{ins=~\"$ins\"}[$__interval])"     ,"format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Silences query count","refId":"A" },
            { "datasource":{"uid":"$datasource"},"expr":"increase(alertmanager_silences_query_errors_total{ins=~\"$ins\"}[$__interval])","format":"time_series",             "intervalFactor":1,"legendFormat":"Silences query fails","refId":"B" }
          ],
          "title":"Silences query count on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":147},
          "id":138,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{ins=~\"$ins\"}[$__interval]))","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Silences query duration","refId":"A"}],
          "title":"Silences query duration on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":153},
          "id":149,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"alertmanager_silences_snapshot_size_bytes{ins=~\"$ins\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Silences snapshot size","refId":"A"}],
          "title":"Silences snapshot size on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":159},
          "id":143,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"alertmanager_silences_snapshot_duration_seconds{ins=~\"$ins\", quantile=\"0.99\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Silences snapshot duration","refId":"A"}],
          "title":"Silences snapshot duration on $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":6,"w":24,"x":0,"y":165},
          "id":131,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[{"datasource":{"uid":"$datasource"},"expr":"alertmanager_silences_gc_duration_seconds{ins=~\"$ins\"}","format":"time_series","hide":false,"intervalFactor":1,"legendFormat":"Silences GC duration","refId":"A"}],
          "title":"Silences GC duration on $ins",
          "type":"timeseries"
        }
      ],
      "title":"Silences",
      "type":"row"
    },
    {
      "collapsed":true,
      "gridPos":{"h":1,"w":24,"x":0,"y":38},
      "id":173,
      "panels":[
        {
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":7,"w":24,"x":0,"y":172},
          "id":175,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            {
              "datasource":{"uid":"$datasource"},
              "expr":"rate(process_cpu_seconds_total{ins=~\"$ins\"}[$__interval])",
              "format":"time_series",
              "groupBy":[{"params":["$__interval"],"type":"time"},{"params":["null"],"type":"fill"}],
              "interval":"",
              "intervalFactor":1,
              "legendFormat":"{{instance}}",
              "orderByTime":"ASC",
              "policy":"default",
              "refId":"A",
              "resultFormat":"time_series",
              "select":[[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],
              "tags":[]
            },
            {"datasource":{"uid":"$datasource"},"expr":"max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$ins\"}) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"Limit {{pod}}","refId":"B"}
          ],
          "title":"CPU usage/s for $ins",
          "type":"timeseries"
        },{
          "datasource":{"uid":"$datasource"},
          "fieldConfig":{
            "defaults":{
              "color":{"mode":"palette-classic"},
              "custom":{
                "axisBorderShow":false,
                "axisCenteredZero":false,
                "axisColorMode":"text",
                "axisLabel":"",
                "axisPlacement":"auto",
                "barAlignment":0,
                "barWidthFactor":0.6,
                "drawStyle":"line",
                "fillOpacity":0,
                "gradientMode":"none",
                "hideFrom":{"legend":false,"tooltip":false,"viz":false},
                "insertNulls":false,
                "lineInterpolation":"linear",
                "lineWidth":1,
                "pointSize":5,
                "scaleDistribution":{"type":"linear"},
                "showPoints":"auto",
                "showValues":false,
                "spanNulls":false,
                "stacking":{"group":"A","mode":"none"},
                "thresholdsStyle":{"mode":"off"}
              },
              "mappings":[],
              "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}
            },
            "overrides":[]
          },
          "gridPos":{"h":7,"w":24,"x":0,"y":179},
          "id":177,
          "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
          "pluginVersion":"12.3.0",
          "repeat":"instance",
          "repeatDirection":"h",
          "targets":[
            { "datasource":{"uid":"$datasource"},"expr":"process_resident_memory_bytes{ins=~\"$ins\"}"                                ,"format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"{{ instance }}" ,"refId":"E" },
            { "datasource":{"uid":"$datasource"},"expr":"max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$ins\"}) by (pod)","format":"time_series",                           "intervalFactor":1,"legendFormat":"Limit {{ pod }}","refId":"A" }
          ],
          "title":"Memory usage for $ins",
          "type":"timeseries"
        }
      ],
      "title":"Resources",
      "type":"row"
    }
  ],
  "preload":false,
  "refresh":"5m",
  "schemaVersion":42,
  "tags":["Pigsty","INFRA","ALERT","INSTANCE"],
  "templating":{
    "list":[
      {"current":{},"hide":2,"includeAll":false,"label":"Prometheus datasource","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},
      {
        "allowCustomValue":true,
        "current":{},
        "datasource":{"type":"prometheus","uid":"ds-prometheus"},
        "definition":"label_values(alertmanager_build_info{cls=\"alertmanager\"},ins)",
        "description":"AlertManager Instance (the `ins` label)",
        "includeAll":false,
        "label":"Instance",
        "name":"ins",
        "options":[],
        "query":{"qryType":1,"query":"label_values(alertmanager_build_info{cls=\"alertmanager\"},ins)","refId":"PrometheusVariableQueryEditor-VariableQuery"},
        "refresh":2,
        "regex":"",
        "sort":1,
        "type":"query"
      },
      {"current":{},"datasource":{"type":"prometheus","uid":"$datasource"},"definition":"label_values(alertmanager_build_info{ins=\"$ins\"},ip)","hide":2,"name":"ip","options":[],"query":{"qryType":1,"query":"label_values(alertmanager_build_info{ins=\"$ins\"},ip)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"}
    ]
  },
  "time":{"from":"now-12h","to":"now"},
  "timepicker":{},
  "timezone":"",
  "title":"Alert Manager",
  "uid":"alertmanager-instance",
  "version":1,
  "weekStart":""
}