{
  "annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},
  "author":"Ruohang Feng (rh@vonng.com)",
  "description":"ETCD Service Overview",
  "editable":true,
  "fiscalYearStartMonth":0,
  "graphTooltip":0,
  "id":null,
  "license":"https://pigsty.io/docs/about/license/",
  "links":[],
  "panels":[
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":118,"panels":[],"title":"Overview","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Members: Total number of instance in this cluster\n\nQuorum : minimal alive instances required, calculated as floor(total members / 2) + 1\n\nTolerance: max failure nodes allowed, calculated as member - quorum",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"noValue":"N/A","thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},"unit":"none"},"overrides":[]},
      "gridPos":{"h":7,"w":3,"x":0,"y":1},
      "id":129,
      "maxDataPoints":100,
      "options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":18,"valueSize":32},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(etcd_up{job=\"etcd\", cls=\"$cls\"})"                                                                          ,"format":"time_series","hide":false,"instant":true,"intervalFactor":2,"legendFormat":"Instance" ,"range":false,"refId":"D","step":600 },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(etcd_server_id{job=\"etcd\", cls=\"$cls\"})"                                                                   ,"format":"time_series","hide":false,"instant":true,"intervalFactor":2,"legendFormat":"Members"  ,"range":false,"refId":"B","step":600 },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"ceil(count(etcd_server_id{job=\"etcd\", cls=\"$cls\"})/ 2 + 0.1)"                                                    ,"format":"time_series",             "instant":true,"intervalFactor":2,"legendFormat":"Quorum"   ,"range":false,"refId":"A","step":600 },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(etcd_server_id{job=\"etcd\", cls=\"$cls\"}) - ceil(count(etcd_server_id{job=\"etcd\", cls=\"$cls\"})/ 2 + 0.1)","format":"time_series","hide":false,"instant":true,"intervalFactor":2,"legendFormat":"Tolerance","range":false,"refId":"C","step":600 }
      ],
      "title":"ETCD Spec",
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"0":{"color":"#cc4637d9","index":0,"text":"Down"},"1":{"color":"#346f36cc","index":1,"text":"Alive"}},"type":"value"}],"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f","value":0}]}},"overrides":[]},
      "gridPos":{"h":7,"w":4,"x":3,"y":1},
      "id":91,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max by (ins,instance) (up{job=\"etcd\", cls=\"$cls\"})","instant":true,"legendFormat":"{{ ins }} @ {{ instance }}","range":false,"refId":"A"}],
      "title":"ETCD Members",
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"All instances among etcd cluster ${cls}\n\nInstance: Goto PGSQL Instance\n\nIP: Goto PGSQL Node\n\nStatus: Goto PGSQL Service\n\nLoad: max(cpu,postgres,pgbouncer)\n\nSpace: Disk space usage max(all device)\n\nProxy: session number, Goto Haproxy Admin Page",
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"custom":{"align":"center","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"mappings":[],"max":1.2,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#e3e3e3e0","value":0}]}},
        "overrides":[
          {"matcher":{"id":"byName","options":"Instance"},"properties":[{"id":"links","value":[]},{"id":"custom.width"}]},
          {
            "matcher":{"id":"byName","options":"Status"},
            "properties":[
              { "id":"custom.width"      ,"value":90                                                                                                                                                                                                                                                 },
              { "id":"mappings"          ,"value":[{"options":{"0":{"color":"#346f36cc","index":0,"text":"Follower"},"1":{"color":"#3e668f","index":1,"text":"Learner"},"2":{"color":"#f79f64","index":2,"text":"Leader"},"3":{"color":"#b783af","index":3,"text":"LeaderLearner"}},"type":"value"}] },
              { "id":"custom.cellOptions","value":{"mode":"gradient","type":"color-background"}                                                                                                                                                                                                      },
              { "id":"custom.align"      ,"value":"center"                                                                                                                                                                                                                                           }
            ]
          },
          {"matcher":{"id":"byName","options":"Version"},"properties":[{"id":"custom.width","value":90}]},{"matcher":{"id":"byName","options":"Server ID"},"properties":[{"id":"custom.width","value":160},{"id":"unit","value":"string"},{"id":"custom.inspect","value":true}]},
          {"matcher":{"id":"byName","options":"Alive"},"properties":[{"id":"custom.width","value":80},{"id":"mappings","value":[{"options":{"0":{"color":"#cc4637d9","index":1,"text":"Dead"},"1":{"color":"#346f36cc","index":0,"text":"Alive"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"type":"color-background"}}]},
          {"matcher":{"id":"byName","options":"IP"},"properties":[{"id":"custom.width","value":120},{"id":"links","value":[{"title":"Node Instance : ${__data.fields.Instance} @ ${__data.fields.IP}","url":"/d/node-instance?var-id=${__data.fields.IP}&${__url_time_range}"}]}]},{"matcher":{"id":"byName","options":"Address"},"properties":[{"id":"custom.width","value":150}]}
        ]
      },
      "gridPos":{"h":7,"w":17,"x":7,"y":1},
      "id":116,
      "options":{"cellHeight":"sm","showHeader":true,"sortBy":[{"desc":false,"displayName":"Instance"}]},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"up{job=\"etcd\", cls=\"$cls\"}"                                                                                             ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"group by (ins,server_version) (etcd_server_version{job=\"etcd\", cls=\"$cls\"})"                                            ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"B" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max by (ins) (etcd_server_is_leader{job=\"etcd\", cls=\"$cls\"} * 2 \n+ etcd_server_is_learner{job=\"etcd\", cls=\"$cls\"})","format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"C" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"etcd_server_id{job=\"etcd\", cls=\"$cls\"}"                                                                                 ,"format":"table","hide":false,"instant":true,"interval":"","legendFormat":"","refId":"D" }
      ],
      "title":"ETCD Member List",
      "transformations":[
        {"id":"seriesToColumns","options":{"byField":"ins"}},
        {
          "id":"organize",
          "options":{
            "excludeByName":{
              "Time":true,
              "Time 1":true,
              "Time 10":true,
              "Time 2":true,
              "Time 3":true,
              "Time 4":true,
              "Time 6":true,
              "Time 8":true,
              "Time 9":true,
              "Value #A":false,
              "Value #B":true,
              "Value #C":false,
              "Value #D":true,
              "Value #E":true,
              "Value #H":false,
              "__name__":true,
              "__name__ 1":true,
              "__name__ 2":true,
              "__name__ 3":true,
              "__name__ 5":true,
              "__name__ 7":true,
              "cls":true,
              "cls 1":true,
              "cls 2":true,
              "cls 3":true,
              "cls 4":true,
              "cls 5":true,
              "cls 6":true,
              "cls 7":true,
              "instance":false,
              "instance 1":true,
              "instance 2":true,
              "instance 3":true,
              "instance 4":true,
              "instance 5":true,
              "instance 6":true,
              "ip 2":true,
              "ip 3":true,
              "ip 4":true,
              "ip 5":true,
              "ip 6":true,
              "ip 7":true,
              "job":true,
              "job 1":true,
              "job 2":true,
              "job 3":true,
              "job 4":true,
              "job 5":true,
              "job 6":true,
              "job 7":true
            },
            "indexByName":{"Time 1":6,"Time 2":7,"Time 3":9,"Time 4":10,"Value #A":5,"Value #B":8,"Value #C":4,"Value #D":20,"__name__ 1":11,"__name__ 2":15,"cls 1":12,"cls 2":16,"ins":0,"instance 1":13,"instance 2":17,"ip 1":2,"ip 2":18,"job 1":14,"job 2":19,"server_id":1,"server_version":3},
            "renameByName":{"Time 4":"","Value #A":"Alive","Value #B":"","Value #C":"Status","Value #D":"","Value #E":"","Value #F":"RT","Value #G":"LB Conn","Value #H":"LB","Value #I":"QPS","Value #J":"LB Clients","Value #K":"Lag","cls 1":"","cls 2":"","ins":"Instance","instance":"Address","instance 2":"","ip":"Node IP","ip 1":"IP","job":"","server_id":"Server ID","server_version":"Version"}
          }
        }
      ],
      "type":"table"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"mappings":[],"noValue":"N/A","thresholds":{"mode":"absolute","steps":[{"color":"red","value":0}]},"unit":"none"},
        "overrides":[
          {
            "matcher":{"id":"byName","options":"ETCD Health"},
            "properties":[
              {
                "id":"mappings",
                "value":[
                  { "options":{"0":{"color":"#b783af","index":0,"text":"All Dead"},"1":{"color":"#346f36cc","index":1,"text":"Healthy"}},"type":"value"   },{ "options":{"match":"null+nan","result":{"color":"#b783af","index":2,"text":"Unknown"}}                              ,"type":"special" },
                  { "options":{"from":0.49999,"result":{"color":"#cc4637d9","index":3,"text":"1 / 2"},"to":0.50001}                     ,"type":"range"   },{ "options":{"from":0.66666,"result":{"color":"#f79f64","index":4,"text":"2 / 3"},"to":0.66667}                       ,"type":"range"   },
                  { "options":{"from":0.33333,"result":{"color":"#b783af","index":5,"text":"1 / 3"},"to":0.33334}                       ,"type":"range"   }
                ]
              },
              {"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#b783af","value":0},{"color":"#cc4637d9","value":0.5},{"color":"#f79f64","value":0.66},{"color":"#346f36cc","value":1}]}},{"id":"unit","value":"percentunit"},{"id":"decimals","value":0}
            ]
          }
        ]
      },
      "gridPos":{"h":6,"w":3,"x":0,"y":8},
      "id":134,
      "maxDataPoints":100,
      "options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{"valueSize":32},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(etcd_server_id{job=\"etcd\", cls=\"$cls\"}) by (cls) /\n count(etcd_server_id{job=\"etcd\", cls=\"$cls\"}) by (cls)\nor on() vector(0)","format":"time_series","instant":true,"intervalFactor":2,"legendFormat":"ETCD Health","range":false,"refId":"A","step":600}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"mappings":[],"noValue":"N/A","thresholds":{"mode":"absolute","steps":[{"color":"red","value":0}]},"unit":"none"},
        "overrides":[{"matcher":{"id":"byName","options":"Has Leader"},"properties":[{"id":"mappings","value":[{"options":{"0":{"color":"#cc4637d9","index":0,"text":"No"},"1":{"color":"#346f36cc","index":1,"text":"Yes"}},"type":"value"},{"options":{"match":"null+nan","result":{"color":"#b783af","index":2,"text":"Unknown"}},"type":"special"}]}]}]
      },
      "gridPos":{"h":2,"w":4,"x":3,"y":8},
      "id":87,
      "maxDataPoints":100,
      "options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{"valueSize":32},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min(etcd_server_has_leader{job=\"etcd\", cls=\"$cls\"}) or on() vector(0)","format":"time_series","instant":true,"intervalFactor":2,"legendFormat":"Has Leader","range":false,"refId":"A","step":600}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"custom":{"axisPlacement":"auto","fillOpacity":70,"hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineWidth":0,"spanNulls":false},"decimals":0,"mappings":[],"min":-1,"thresholds":{"mode":"absolute","steps":[{"color":"#cc4637d9","value":0},{"color":"#346f36cc","value":1}]}},
        "overrides":[{"matcher":{"id":"byName","options":"ETCD Cluster Healthy"},"properties":[{"id":"mappings","value":[{"options":{"0":{"color":"#b783af","index":1,"text":"\ud83d\udd25NOLEADER"},"1":{"color":"#fcdb72","index":0,"text":"OK"}},"type":"value"}]},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#cc4637d9","value":0},{"color":"#3e668f","value":1}]}}]}]
      },
      "gridPos":{"h":6,"w":17,"x":7,"y":8},
      "id":132,
      "options":{"alignValue":"left","legend":{"displayMode":"list","placement":"bottom","showLegend":false},"mergeValues":true,"rowHeight":0.96,"showValue":"never","tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"min(etcd_server_has_leader{job=\"etcd\", cls=\"$cls\"}) or on() vector(0)","hide":false,"legendFormat":"ETCD Cluster Healthy","range":true,"refId":"D" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"up{job=\"etcd\", cls=\"$cls\"}"                                           ,             "legendFormat":"{{ ins }} @ {{ ip }}","range":true,"refId":"A" }
      ],
      "title":"ETCD Aliveness",
      "type":"state-timeline"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Very Important Metric: This limit defines the maximum amount of data (in bytes) that can be stored in the ETCD database. When the quota is reached, further write operations will be rejected until space is freed or the quota is increased.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"thresholds"},
          "links":[{"title":"Quota Usage Details","url":"/d/etcd-overview?var-cls=${cls}&viewPanel=138&${__url_time_range}"}],
          "mappings":[],
          "min":0,
          "noValue":"\u2205",
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0},{"color":"#f79f64","value":0.8},{"color":"red","value":0.9},{"color":"#b783af","value":0.95},{"color":"text","value":1}]},
          "unit":"percentunit"
        },
        "overrides":[]
      },
      "gridPos":{"h":4,"w":4,"x":3,"y":10},
      "id":164,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"etcd:cls:quota_usage{cls=\"$cls\"}","instant":false,"interval":"","legendFormat":"Quota Usage","range":true,"refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "axisSoftMin":0,
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":20,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"stepBefore",
            "lineWidth":0,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"auto",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "decimals":0,
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]}
        },
        "overrides":[]
      },
      "gridPos":{"h":4,"w":7,"x":0,"y":14},
      "id":135,
      "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"etcd_server_leader_changes_seen_total{job=\"etcd\", cls=\"$cls\"}","instant":false,"legendFormat":"{{ ins }} @ {{ instance }}","range":true,"refId":"A"}],
      "title":"Leader Elections",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"The current revision\n\nhttps://etcd.io/docs/v3.5/learning/data_model/",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"links":[],"mappings":[{"options":{"match":"null+nan","result":{"color":"gray","index":0,"text":"\u2205"}},"type":"special"}],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},"unit":"none"},"overrides":[]},
      "gridPos":{"h":4,"w":3,"x":7,"y":14},
      "id":150,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max(etcd_debugging_mvcc_current_revision{cls=\"$cls\"})"                                                     ,             "interval":"","legendFormat":"Revision Current","range":true,"refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max(etcd_debugging_mvcc_compact_revision{cls=\"$cls\"})"                                                     ,"hide":false,"interval":"","legendFormat":"Revision Compact","range":true,"refId":"B" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max(etcd_debugging_mvcc_current_revision{cls=\"$cls\"} - etcd_debugging_mvcc_compact_revision{cls=\"$cls\"})","hide":false,"interval":"","legendFormat":"Revision Diff"   ,"range":true,"refId":"C" }
      ],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Reads per minute",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"links":[],"mappings":[{"options":{"match":"null+nan","result":{"color":"gray","index":0,"text":"\u2205"}},"type":"special"}],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},"unit":"none"},"overrides":[]},
      "gridPos":{"h":4,"w":2,"x":10,"y":14},
      "id":161,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(etcd:ins:rpm{cls=\"$cls\"})","interval":"","legendFormat":"Reads/m","range":true,"refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Reads per minute",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"links":[],"mappings":[{"options":{"match":"null+nan","result":{"color":"gray","index":0,"text":"\u2205"}},"type":"special"}],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},"unit":"none"},"overrides":[]},
      "gridPos":{"h":4,"w":2,"x":12,"y":14},
      "id":162,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(etcd:ins:wpm{cls=\"$cls\"})","interval":"","legendFormat":"Writes/m","range":true,"refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"This metric helps monitor the latency of fsync operations, specifically focusing on the worst-case scenario (99th percentile) over the recent 5 minutes, allowing for better performance analysis and optimization of ETCD's disk I/O operations.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"thresholds"},
          "links":[],
          "mappings":[{"options":{"match":"null+nan","result":{"color":"gray","index":0,"text":"\u2205"}},"type":"special"},{"options":{"from":7,"result":{"color":"text","index":1,"text":"\u221e"},"to":9999999},"type":"range"}],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0},{"color":"#f79f64","value":0.1},{"color":"#cc4637d9","value":0.2},{"color":"#b783af","value":0.5},{"color":"text","value":1}]},
          "unit":"s"
        },
        "overrides":[]
      },
      "gridPos":{"h":4,"w":2,"x":14,"y":14},
      "id":165,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"avg(etcd:ins:network_peer_rt_p95_5m{job=\"etcd\", cls=\"$cls\"})","hide":false,"instant":false,"interval":"","legendFormat":"Network RT","range":true,"refId":"B"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"This metric helps monitor the latency of fsync operations, specifically focusing on the worst-case scenario (99th percentile) over the recent 5 minutes, allowing for better performance analysis and optimization of ETCD's disk I/O operations.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"thresholds"},
          "links":[],
          "mappings":[{"options":{"match":"null+nan","result":{"color":"gray","index":0,"text":"\u2205"}},"type":"special"},{"options":{"from":7,"result":{"color":"text","index":1,"text":"\u221e"},"to":9999999},"type":"range"}],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0},{"color":"#f79f64","value":0.01},{"color":"#cc4637d9","value":0.05},{"color":"#b783af","value":0.1},{"color":"text","value":1}]},
          "unit":"s"
        },
        "overrides":[]
      },
      "gridPos":{"h":4,"w":2,"x":16,"y":14},
      "id":156,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"avg(etcd:ins:wal_fsync_rt_p95_5m{job=\"etcd\", cls=\"$cls\"})","hide":false,"instant":false,"interval":"","legendFormat":"Fsync RT","range":true,"refId":"B"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Database Size",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"links":[],"mappings":[{"options":{"match":"null+nan","result":{"color":"gray","index":0,"text":"\u2205"}},"type":"special"}],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},"unit":"none"},"overrides":[]},
      "gridPos":{"h":4,"w":2,"x":18,"y":14},
      "id":166,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max by (cls) (etcd_debugging_mvcc_keys_total{cls=\"$cls\"})","interval":"","legendFormat":"Keys","range":true,"refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Database Size",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"links":[],"mappings":[{"options":{"match":"null+nan","result":{"color":"gray","index":0,"text":"\u2205"}},"type":"special"}],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},"unit":"bytes"},"overrides":[]},
      "gridPos":{"h":4,"w":2,"x":20,"y":14},
      "id":153,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max by (cls) (etcd_mvcc_db_total_size_in_bytes{cls=\"$cls\"})","interval":"","legendFormat":"DB Size","range":true,"refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"The storage quote size of this etcd cluster, if full, etcd will not accept any write operation.",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"links":[],"mappings":[{"options":{"match":"null+nan","result":{"color":"gray","index":0,"text":"\u2205"}},"type":"special"}],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},"unit":"bytes"},"overrides":[]},
      "gridPos":{"h":4,"w":2,"x":22,"y":14},
      "id":152,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"value_and_name","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max(etcd_server_quota_backend_bytes{cls=\"$cls\"})","interval":"","legendFormat":"Quota","range":true,"refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{
        "defaults":{"color":{"mode":"thresholds"},"decimals":0,"links":[],"mappings":[],"max":1,"min":0,"noValue":"OK","thresholds":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":4},{"color":"#b783af","value":8},{"color":"text","value":16}]},"unit":"none"},
        "overrides":[{"matcher":{"id":"byFrameRefID","options":"A"},"properties":[{"id":"links","value":[{"targetBlank":true,"title":"Pigsty Alertmanager","url":"/alertmgr/"}]}]}]
      },
      "gridPos":{"h":4,"w":3,"x":0,"y":18},
      "id":158,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(ALERTS{alertstate=\"firing\", cls=\"$cls\", job=\"etcd\"}) or on() vector(0)",             "interval":"","legendFormat":"Firing Alert" ,"range":true,"refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(ALERTS{alertstate=\"pending\", cls=\"$cls\", job=\"etcd\"})"                 ,"hide":false,"interval":"","legendFormat":"Pending Alert","range":true,"refId":"B" }
      ],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"thresholds"},
          "decimals":0,
          "links":[{"targetBlank":true,"title":"AlertInfo ${__field.labels.alertname}","url":"/alertmgr/#/alerts?filter=%7Balertname%3D%22${__field.labels.alertname}%22%7D"}],
          "mappings":[],
          "max":1,
          "min":0,
          "noValue":"OK",
          "thresholds":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":2},{"color":"#b783af","value":4}]},
          "unit":"none"
        },
        "overrides":[]
      },
      "gridPos":{"h":4,"w":4,"x":3,"y":18},
      "id":157,
      "options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(ALERTS{job=\"etcd\", cls=\"etcd\"}) by (alertname, severity)","interval":"","legendFormat":"{{ alertname }}","range":true,"refId":"A"}],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Alerting event graph. Each stripe represent an alerting Event. Gray transparent stipe are pending alerts.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"hidden",
            "axisSoftMax":1.2,
            "axisSoftMin":0,
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":70,
            "gradientMode":"none",
            "hideFrom":{"graph":false,"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"stepBefore",
            "lineWidth":0,
            "pointSize":1,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"normal"},
            "thresholdsStyle":{"mode":"off"}
          },
          "decimals":0,
          "links":[
            { "targetBlank":true,"title":"AlertInfo ${__field.labels.alertname} on Cluster ${cls}","url":"/alertmgr/#/alerts?filter=%7Bcls%3D%22${cls}%22%2C%20alertname%3D%22${__field.labels.alertname}%22%7D"       },
            { "targetBlank":true,"title":"Silence ${__field.labels.alertname} on Cluster ${cls}"  ,"url":"/alertmgr/#/silences/new?filter=%7Bcls%3D%22${cls}%22%2C%20alertname%3D%22${__field.labels.alertname}%22%7D" },
            {                    "title":"PGSQL Instance for ${__field.labels.ins}"               ,"url":"/d/pgsql-instance?var-ins=${__field.labels.ins}&${__url_time_range}"                                         },
            {                    "title":"PGSQL Node for ${__field.labels.ins}"                   ,"url":"/d/node-instance?var-ins=${__field.labels.ins}&${__url_time_range}"                                          }
          ],
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"none"
        },
        "overrides":[
          { "matcher":{"id":"byFrameRefID","options":"A"},"properties":[{"id":"custom.fillOpacity","value":70},{"id":"mappings","value":[{"options":{"1":{"index":0,"text":"Firing\ud83d\udd25"}},"type":"value"}]}]                                                                                        },
          { "matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"custom.fillOpacity","value":30},{"id":"color","value":{"fixedColor":"rgba(128, 128, 128, 0.5)","mode":"fixed"}},{"id":"mappings","value":[{"options":{"1":{"color":"gray","index":0,"text":"Pend\u23f0"}},"type":"value"}]}] },
          { "matcher":{"id":"byName","options":"Alert"}  ,"properties":[{"id":"color","value":{"fixedColor":"#cc4637d9","mode":"fixed"}}]                                                                                                                                                                   },
          { "matcher":{"id":"byType","options":"time"}   ,"properties":[{"id":"custom.axisPlacement","value":"auto"}]                                                                                                                                                                                       }
        ]
      },
      "gridPos":{"h":4,"w":17,"x":7,"y":18},
      "id":124,
      "options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Last","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"ALERTS{cls=\"$cls\", alertstate=\"firing\"}"  ,             "interval":"","legendFormat":"[{{ severity }}\ud83d\udd25] {{alertname}} @ {{ ins }}","refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"ALERTS{cls=\"$cls\", alertstate=\"pending\"} ","hide":false,"interval":"","legendFormat":"[{{ severity }}\u23f0] {{alertname}} @ {{ ins }}"      ,"refId":"B" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"0"                                            ,"hide":false,"interval":"","legendFormat":"Alert"                                                 ,"refId":"C" }
      ],
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":128,"panels":[],"title":"Metrics","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"This limit defines the maximum amount of data (in bytes) that can be stored in the ETCD database. When the quota is reached, further write operations will be rejected until space is freed or the quota is increased.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":2,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"percentunit"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":12,"x":0,"y":23},
      "id":138,
      "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"etcd_mvcc_db_total_size_in_bytes{job=\"etcd\", cls=\"$cls\"} / etcd_server_quota_backend_bytes{job=\"etcd\", cls=\"$cls\"}","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{ ins }}","metric":"","range":true,"refId":"A","step":4}],
      "title":"Quota Usage",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"In ETCD, revision refers to the current version of the data stored in the cluster, incrementing with each write operation. It allows users to track changes and retrieve specific versions of keys.\n\nCompacted revision is the lowest revision number that has been retained after the compaction process, which removes old, obsolete data versions to free up space. It represents the point after which all data is considered stable and accessible, ensuring efficient storage management.",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":2,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"none"
        },
        "overrides":[{"matcher":{"id":"byName","options":"Current"},"properties":[{"id":"color","value":{"fixedColor":"#5b9cd5","mode":"fixed"}},{"id":"custom.fillBelowTo","value":"Compact"},{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byName","options":"Compact"},"properties":[{"id":"color","value":{"fixedColor":"#346f36cc","mode":"fixed"}}]}]
      },
      "gridPos":{"h":6,"w":8,"x":12,"y":23},
      "id":159,
      "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max by (cls) (etcd_debugging_mvcc_current_revision{job=\"etcd\", cls=\"$cls\"})",             "intervalFactor":2,"legendFormat":"Current","metric":"process_resident_memory_bytes","range":true,"refId":"A","step":4 },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max by (cls) (etcd_debugging_mvcc_compact_revision{job=\"etcd\", cls=\"$cls\"})","hide":false,"intervalFactor":2,"legendFormat":"Compact","metric":"process_resident_memory_bytes","range":true,"refId":"B","step":4 }
      ],
      "title":"Revision & Compact",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"When were last mvcc ddb compaction happen: https://etcd.io/docs/v3.4/op-guide/maintenance/",
      "fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"noValue":"N/A","thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},"unit":"dateTimeAsIso"},"overrides":[{"matcher":{"id":"byName","options":"Elapse"},"properties":[{"id":"unit","value":"s"},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#f79f64","value":0},{"color":"#346f36cc","value":120}]}}]}]},
      "gridPos":{"h":6,"w":4,"x":20,"y":23},
      "id":139,
      "maxDataPoints":100,
      "options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12,"valueSize":18},"textMode":"auto","wideLayout":true},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min by (cls) (etcd_debugging_mvcc_db_compaction_last{cls=\"$cls\"}) * 1000"  ,"format":"time_series","hide":false,"instant":true,"intervalFactor":2,"legendFormat":"Last Compact","range":false,"refId":"B","step":600 },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"time() - min by (cls) (etcd_debugging_mvcc_db_compaction_last{cls=\"$cls\"})","format":"time_series","hide":false,"instant":true,"intervalFactor":2,"legendFormat":"Elapse"      ,"range":false,"refId":"A","step":600 }
      ],
      "type":"stat"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Quota: backend limit\n\nTotal: Actual Database Size\n\nin Use: Estimated Compacted Size",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":2,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"bytes"
        },
        "overrides":[
          {                                "matcher":{"id":"byName","options":"Quota"}                                                                             ,"properties":[{"id":"color","value":{"fixedColor":"semi-dark-red","mode":"fixed"}}]                                       },
          {                                "matcher":{"id":"byName","options":"Total"}                                                                             ,"properties":[{"id":"color","value":{"fixedColor":"#3e668f","mode":"fixed"}},{"id":"custom.fillBelowTo","value":"InUse"}] },
          { "__systemRef":"hideSeriesFrom","matcher":{"id":"byNames","options":{"mode":"exclude","names":["InUse","Total"],"prefix":"All except:","readOnly":true}},"properties":[{"id":"custom.hideFrom","value":{"legend":false,"tooltip":true,"viz":true}}]                                },
          {                                "matcher":{"id":"byName","options":"InUse"}                                                                             ,"properties":[{"id":"color","value":{"fixedColor":"#357037d1","mode":"fixed"}},{"id":"custom.fillOpacity","value":30}]    }
        ]
      },
      "gridPos":{"h":6,"w":12,"x":0,"y":29},
      "id":101,
      "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max by (cls) (etcd_mvcc_db_total_size_in_use_in_bytes{job=\"etcd\", cls=\"$cls\"} )","hide":false,"interval":"","intervalFactor":2,"legendFormat":"InUse","metric":"","range":true,"refId":"A","step":4 },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"max by (cls) (etcd_mvcc_db_total_size_in_bytes{job=\"etcd\", cls=\"$cls\"})"        ,"hide":false,"interval":"","intervalFactor":2,"legendFormat":"Total","metric":"","range":true,"refId":"B","step":4 },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"min by (cls) (etcd_server_quota_backend_bytes{job=\"etcd\", cls=\"$cls\"})"         ,"hide":false,"interval":"","intervalFactor":2,"legendFormat":"Quota","metric":"","range":true,"refId":"C","step":4 }
      ],
      "title":"Database Size",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":2,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0},{"color":"red","value":80}]},
          "unit":"bytes"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":12,"x":12,"y":29},
      "id":103,
      "options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"process_resident_memory_bytes{job=\"etcd\", cls=\"$cls\"}","intervalFactor":2,"legendFormat":"{{instance}} Resident Memory","metric":"process_resident_memory_bytes","range":true,"refId":"A","step":4}],
      "title":"Memory",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"\u25b2 for read, \u25bc for write",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":true,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"opacity",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"smooth",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0},{"color":"red","value":80}]},
          "unit":"short"
        },
        "overrides":[{"matcher":{"id":"byName","options":"/dfa/"},"properties":[{"id":"custom.fillOpacity","value":50},{"id":"custom.lineWidth","value":2}]}]
      },
      "gridPos":{"h":6,"w":12,"x":0,"y":35},
      "id":167,
      "options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"sum(etcd:ins:rpm{cls=\"$cls\"})"  ,"interval":"","legendFormat":"\u25b2 Reads" ,"range":true,"refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"- sum(etcd:ins:wpm{cls=\"$cls\"})","interval":"","legendFormat":"\u25bc Writes","range":true,"refId":"B" }
      ],
      "title":"Read / Write per Mintue",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"etcd network peer rt (p95 in last 5m)",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "axisSoftMax":0.05,
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"auto",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"area"}
          },
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"yellow","value":0.1},{"color":"red","value":0.2}]},
          "unit":"s"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":12,"x":12,"y":35},
      "id":114,
      "options":{"legend":{"calcs":["max"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":" histogram_quantile(0.95, rate(etcd_network_peer_round_trip_time_seconds_bucket{cls=\"$cls\", job=\"etcd\"}[5m]))","legendFormat":"{{ ins }} -> {{ To }}","range":true,"refId":"A"}],
      "title":"Network RT (p95 5m)",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"etcd disk wal fsync duration (p95 in last 5m)",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"stepAfter",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"area"}
          },
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"orange","value":0.01},{"color":"red","value":0.05},{"color":"purple","value":0.1}]},
          "unit":"s"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":12,"x":0,"y":41},
      "id":99,
      "options":{"legend":{"calcs":["max"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"histogram_quantile(0.95, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"etcd\", cls=\"$cls\"}[5m])) by (ins, le))","hide":false,"intervalFactor":2,"legendFormat":"{{ ins }}","metric":"etcd_disk_wal_fsync_duration_seconds_bucket","range":true,"refId":"A","step":4}],
      "title":"WAL Fsync RT (p95 5m)",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"\u25b2 for Sent, \u25bc for receive",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":50,
            "gradientMode":"opacity",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"smooth",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"Bps"
        },
        "overrides":[{"matcher":{"id":"byName","options":"/dfa/"},"properties":[{"id":"custom.fillOpacity","value":40},{"id":"custom.fillOpacity","value":10},{"id":"custom.lineWidth","value":2}]}]
      },
      "gridPos":{"h":6,"w":12,"x":12,"y":41},
      "id":111,
      "options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"sum(rate(etcd_network_peer_sent_bytes_total{job=\"etcd\", cls=\"$cls\"}[1m])) by (ins)"             ,"interval":"","legendFormat":"\u25b2{{ ins }}","range":true,"refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"- sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"etcd\", cls=\"$cls\"}[1m])) by (ins)","interval":"","legendFormat":"\u25bc{{ ins }}","range":true,"refId":"B" }
      ],
      "title":"Peer Traffic",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"etcd backend commit rt (p95 in last 5m)",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":0,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"stepAfter",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"area"}
          },
          "mappings":[],
          "min":0,
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"orange","value":0.01},{"color":"red","value":0.05},{"color":"purple","value":0.1}]},
          "unit":"s"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":12,"x":0,"y":47},
      "id":160,
      "options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"histogram_quantile(0.95, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"etcd\"}[5m]))","hide":false,"intervalFactor":2,"legendFormat":"{{ ins }}","metric":"etcd_disk_wal_fsync_duration_seconds_bucket","range":true,"refId":"A","step":4}],
      "title":"Backend Commit RT (p95 5m)",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"\u25b2 for Sent, \u25bc for receive",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":30,
            "gradientMode":"opacity",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"smooth",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},
          "unit":"Bps"
        },
        "overrides":[{"matcher":{"id":"byName","options":"/dfa/"},"properties":[{"id":"custom.fillOpacity","value":40},{"id":"custom.fillOpacity","value":10},{"id":"custom.lineWidth","value":2}]}]
      },
      "gridPos":{"h":6,"w":12,"x":12,"y":47},
      "id":110,
      "options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"etcd\", cls=\"$cls\"}[1m])) by (ins)","interval":"","legendFormat":"\u25b2{{ ins }}","range":true,"refId":"A" },
        { "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":true,"expr":"- sum(rate(etcd_network_peer_received_bytes_total{job=\"etcd\", cls=\"$cls\"}[1m])) by (ins)" ,"interval":"","legendFormat":"\u25bc{{ ins }}","range":true,"refId":"B" }
      ],
      "title":"Client Traffic",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":12,"x":0,"y":53},
      "id":89,
      "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"rate(etcd_server_proposals_committed_total{job=\"etcd\", cls=\"$cls\"}[1m])","format":"time_series","intervalFactor":2,"legendFormat":"{{ ins }}","metric":"","range":true,"refId":"A","step":60}],
      "title":"Proposal Committed",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":12,"x":12,"y":53},
      "id":108,
      "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"rate(etcd_server_proposals_applied_total{job=\"etcd\", cls=\"$cls\"}[1m])","format":"time_series","intervalFactor":2,"legendFormat":"{{ ins }}","metric":"","range":true,"refId":"A","step":60}],
      "title":"Proposal Applied",
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":59},"id":137,"panels":[],"title":"MVCC","type":"row"},
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total number of RANGE seen by this etcd instance",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":7,"w":12,"x":0,"y":60},
      "id":146,
      "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"rate(etcd_mvcc_range_total{job=\"etcd\", cls=\"$cls\"}[1m])","format":"time_series","intervalFactor":2,"legendFormat":"{{ ins }}","metric":"","range":true,"refId":"A","step":60}],
      "title":"RANGE Rate1m",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total number of PUT seen by this etcd instance",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":7,"w":12,"x":12,"y":60},
      "id":145,
      "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"rate(etcd_mvcc_delete_total{job=\"etcd\", cls=\"$cls\"}[1m])","format":"time_series","intervalFactor":2,"legendFormat":"{{ ins }}","metric":"","range":true,"refId":"A","step":60}],
      "title":"DELETE Rate1m",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"Total number of PUT seen by this etcd instance",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":7,"w":12,"x":0,"y":67},
      "id":143,
      "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"rate(etcd_mvcc_put_total{job=\"etcd\", cls=\"$cls\"}[1m])","format":"time_series","intervalFactor":2,"legendFormat":"{{ ins }}","metric":"","range":true,"refId":"A","step":60}],
      "title":"PUT Rate1m",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"prometheus","uid":"ds-prometheus"},
      "description":"ETCD MVCC Event: PUT | DELETE | WATCH",
      "fieldConfig":{
        "defaults":{
          "color":{"mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"line",
            "fillOpacity":10,
            "gradientMode":"none",
            "hideFrom":{"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":false,
            "stacking":{"group":"A","mode":"none"},
            "thresholdsStyle":{"mode":"off"}
          },
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"short"
        },
        "overrides":[]
      },
      "gridPos":{"h":7,"w":12,"x":12,"y":67},
      "id":144,
      "options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","expr":"rate(etcd_debugging_mvcc_events_total{job=\"etcd\", cls=\"$cls\"}[1m])","format":"time_series","intervalFactor":2,"legendFormat":"{{ ins }}","metric":"","range":true,"refId":"A","step":60}],
      "title":"MVCC Event Rate 1m",
      "type":"timeseries"
    },
    {"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":74},"id":120,"panels":[],"title":"Logs","type":"row"},
    {
      "datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
      "description":"Etcd logs of this etcd cluster $cls",
      "fieldConfig":{
        "defaults":{
          "color":{"fixedColor":"#346f36cc","mode":"palette-classic"},
          "custom":{
            "axisBorderShow":false,
            "axisCenteredZero":false,
            "axisColorMode":"text",
            "axisLabel":"",
            "axisPlacement":"auto",
            "barAlignment":0,
            "barWidthFactor":0.6,
            "drawStyle":"bars",
            "fillOpacity":100,
            "gradientMode":"none",
            "hideFrom":{"graph":false,"legend":false,"tooltip":false,"viz":false},
            "insertNulls":false,
            "lineInterpolation":"linear",
            "lineWidth":1,
            "pointSize":5,
            "scaleDistribution":{"type":"linear"},
            "showPoints":"never",
            "showValues":false,
            "spanNulls":true,
            "stacking":{"group":"A","mode":"normal"},
            "thresholdsStyle":{"mode":"off"}
          },
          "decimals":0,
          "mappings":[],
          "thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
          "unit":"none"
        },
        "overrides":[]
      },
      "gridPos":{"h":6,"w":24,"x":0,"y":75},
      "id":136,
      "interval":"1s",
      "options":{"legend":{"calcs":["sum"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog ip:in(${members:doublequote}) app:etcd | stats by (ins) count() as count","legendFormat":"{{ ins }}","queryType":"statsRange","refId":"A"}],
      "title":"ETCD Logs",
      "type":"timeseries"
    },
    {
      "datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
      "gridPos":{"h":18,"w":24,"x":0,"y":81},
      "id":122,
      "options":{"dedupStrategy":"none","enableInfiniteScrolling":false,"enableLogDetails":true,"fontSize":"small","prettifyLogMessage":false,"showCommonLabels":false,"showControls":true,"showLabels":false,"showTime":true,"sortOrder":"Descending","syntaxHighlighting":true,"wrapLogMessage":false},
      "pluginVersion":"12.3.0",
      "targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog ip:in(${members:doublequote}) app:etcd","instant":false,"legendFormat":"{{ ins }}","queryType":"instant","range":true,"refId":"A"}],
      "type":"logs"
    }
  ],
  "preload":false,
  "refresh":"",
  "schemaVersion":42,
  "tags":["Pigsty","ETCD","Overview","HOME"],
  "templating":{
    "list":[
      {"current":{},"datasource":{"type":"prometheus","uid":"ds-prometheus"},"definition":"label_values(etcd_cluster_version, cls)","description":"etcd cluster name","includeAll":false,"label":"Cluster","name":"cls","options":[],"query":{"query":"label_values(etcd_cluster_version, cls)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","type":"query"},
      {"current":{},"definition":"label_values(etcd_cluster_version{cls=\"$cls\"},ip)","description":"IP address of this cluster members","hide":2,"includeAll":true,"label":"Members","multi":true,"name":"members","options":[],"query":{"qryType":1,"query":"label_values(etcd_cluster_version{cls=\"$cls\"},ip)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"}
    ]
  },
  "time":{"from":"now-1h","to":"now"},
  "timepicker":{"refresh_intervals":["2s","5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"]},
  "timezone":"browser",
  "title":"Etcd Overview",
  "uid":"etcd-overview",
  "version":1,
  "weekStart":""
}