ai-workspace-infra/observability.svc.plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
280764fe9c
observability.svc.plus/files/grafana/grafana.py

471 lines
15 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
#==============================================================#
# File : grafana.py
# Desc : dump/load/init grafana dashboards
# Ctime : 2022-11-23
# Mtime : 2026-01-29
# Path : files/grafana/grafana.py
# License : Apache-2.0 @ https://pigsty.io/docs/about/license/
# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com)
#==============================================================#
import os, sys, json, requests
def env_flag(name, default):
value = os.environ.get(name)
if value is None:
return default
return value.lower() in ('1', 'true', 'yes', 'on')
# grafana access info
ENDPOINT = os.environ.get("GRAFANA_ENDPOINT", 'http://i.pigsty/ui')
USERNAME = os.environ.get("GRAFANA_USERNAME", 'admin')
PASSWORD = os.environ.get("GRAFANA_PASSWORD", 'pigsty')
CREATE_FOLDERS = env_flag('GRAFANA_CREATE_FOLDERS', True)
SKIP_SUBFOLDERS = env_flag('GRAFANA_SKIP_SUBFOLDERS', False)
FOLDER_TITLES = {
'01-iaas-compute': 'IAAS / 计算',
'02-iaas-storage': 'IAAS / 存储',
'03-iaas-network': 'IAAS / 网络',
'11-paas-control-plane': 'PaaS / 平台控制面',
'12-paas-cluster': 'PaaS / 集群',
'13-paas-db': 'PaaS / DB',
'14-paas-cache': 'PaaS / 缓存',
'15-paas-queue': 'PaaS / 队列',
'21-bu-dns': '业务单元 / DNS',
'22-bu-proxy': '业务单元 / 代理',
'23-bu-gateway': '业务单元 / 网关',
'24-bu-request': '业务单元 / 请求',
'25-bu-throughput': '业务单元 / 吞吐',
}
FOLDER_TAGS = {
'01-iaas-compute': ['IAAS', 'IAAS-COMPUTE'],
'02-iaas-storage': ['IAAS', 'IAAS-STORAGE'],
'03-iaas-network': ['IAAS', 'IAAS-NETWORK'],
'11-paas-control-plane': ['PAAS', 'PAAS-CONTROL-PLANE'],
'12-paas-cluster': ['PAAS', 'PAAS-CLUSTER'],
'13-paas-db': ['PAAS', 'PAAS-DB'],
'14-paas-cache': ['PAAS', 'PAAS-CACHE'],
'15-paas-queue': ['PAAS', 'PAAS-QUEUE'],
'21-bu-dns': ['BU', 'BU-DNS'],
'22-bu-proxy': ['BU', 'BU-PROXY'],
'23-bu-gateway': ['BU', 'BU-GATEWAY'],
'24-bu-request': ['BU', 'BU-REQUEST'],
'25-bu-throughput': ['BU', 'BU-THROUGHPUT'],
}
METADB_PASSWORD = 'DBUser.Viewer'
DEFAULT_DATASOURCES = {
'ds-prometheus': {'uid': 'ds-prometheus', 'orgId': 1, 'name': 'Prometheus', 'type': 'prometheus', 'typeName': 'Prometheus', 'typeLogoUrl': 'public/app/plugins/datasource/prometheus/img/prometheus_logo.svg', 'access': 'proxy',
'url': 'http://127.0.0.1:9058', 'user': '', 'database': '', 'basicAuth': False, 'isDefault': True, 'jsonData': {'queryTimeout': '60s', 'timeInterval': '2s', 'tlsAuth': False, 'tlsAuthWithCACert': False}, 'readOnly': False},
'ds-meta': {'uid': 'ds-meta', 'orgId': 1, 'name': 'Meta', 'type': 'postgres', 'typeName': 'PostgreSQL', 'access': 'proxy', 'url': '127.0.0.1:5432', 'user': 'dbuser_view', 'database': 'meta', 'basicAuth': False, 'isDefault': False, 'readOnly': True,
'jsonData': {'connMaxLifetime': 14400, 'maxIdleConns': 10, 'maxOpenConns': 64, 'postgresVersion': 1500, 'sslmode': 'require', 'tlsAuth': False, 'tlsAuthWithCACert': False}, 'secureJsonData': { 'password': METADB_PASSWORD }},
'ds-vlogs': {'uid': 'ds-vlogs', 'orgId': 1, 'name': 'Loki', 'type': 'loki', 'typeName': 'Loki', 'access': 'proxy', 'url': 'http://127.0.0.1:3100', 'basicAuth': False, 'isDefault': False, 'jsonData': {}, 'readOnly': False}}
##########################################
# generic api
##########################################
def get(path):
return requests.get(
"%s/api/%s" % (ENDPOINT, path),
auth=requests.auth.HTTPBasicAuth(USERNAME, PASSWORD),
headers={'Content-Type': 'application/json'}
).json()
def post(path, payload={}):
return requests.post(
"%s/api/%s" % (ENDPOINT, path),
auth=requests.auth.HTTPBasicAuth(USERNAME, PASSWORD),
headers={'Content-Type': 'application/json'},
json=payload
).json()
def put(path, payload={}):
return requests.put(
"%s/api/%s" % (ENDPOINT, path),
auth=requests.auth.HTTPBasicAuth(USERNAME, PASSWORD),
headers={'Content-Type': 'application/json'},
json=payload
).json()
def delete(path):
return requests.delete(
"%s/api/%s" % (ENDPOINT, path),
auth=requests.auth.HTTPBasicAuth(USERNAME, PASSWORD),
headers={'Content-Type': 'application/json'}
).json()
##########################################
# grafana api
##########################################
# Dashboard ----------------->
def get_dashboard(uid):
return get('dashboards/uid/%s' % uid)
def add_dashboard(d, folder=None):
"""put raw dashboard"""
d["id"] = None
payload = {"dashboard": d, "overwrite": True}
if CREATE_FOLDERS:
if folder is not None and folder != "":
payload["folderUid"] = folder
else:
payload["folderId"] = 0
return post('dashboards/db', payload)
def del_dashboard(uid):
return delete('dashboards/uid/%s' % uid)
def list_dashboards():
return get('search')
def list_datasources():
return get('datasources')
def get_dashboard_id_by_uid(uid):
return get_dashboard(uid)["dashboard"]["id"]
def star_dashboard(id):
return post('user/stars/dashboard/%s' % id)
def star_dashboard_by_uid(uid):
return post('user/stars/dashboard/%s' % get_dashboard_id_by_uid(uid))
# Folder ----------------->
def list_folders():
return get('folders')
def add_folder(uid, title=""):
if not CREATE_FOLDERS:
return
if title == "":
title = resolve_folder_title(uid)
post('folders', {"uid": uid, "title": title})
return put('folders/%s' % uid, {"title": title, "overwrite": True})
def del_folder(uid):
try:
return delete('folders/%s' % uid)
except:
return {}
# Datasource ----------------->
def del_datasource_by_name(name):
return delete('datasources/name/%s' % name)
def get_datasource_id_by_name(name):
return get('datasources/id/%s' % name).get('id')
def create_datasource(ds):
return post('datasources', ds)
def update_datasource(uid, ds):
return put('datasources/uid/%s'% uid, ds)
def ds_query(dsID, query):
return post('ds/query', {
"queries": [
{
"refId": "A",
"datasourceId": dsID,
"rawSql": query,
"format": "table"
}
]
})
def ds_query_by_name(name, query):
dsID = get('datasources/id/%s' % name).get('id')
ds_query(dsID, query)
# Preference ----------------->
def update_org_preference(home="pigsty", theme="light"):
home_id = get_dashboard_id_by_uid(home)
put('org/preferences', {
"theme": theme,
"homeDashboardId": home_id
})
def update_user_preference(home="pigsty", theme="light"):
home_id = get_dashboard_id_by_uid(home)
put('user/preferences', {
"theme": theme,
"homeDashboardId": home_id
})
# Organization ----------------->
def update_orgname(id, new_name):
return put('orgs/%s' % id, {"name": new_name})
##########################################
# process dashboard
##########################################
def dashboard_raw(d):
"""extract raw definition of dashboard"""
raw = d["dashboard"]
if "id" in raw and raw["id"] is not None:
raw["id"] = None
if "templating" in raw and "list" in raw["templating"]:
i = 0
for item in raw["templating"]["list"]:
if "current" in item:
raw["templating"]["list"][i]["current"] = {}
i = i + 1
return raw
def load_dashboard(path, substitute=False):
if substitute:
with open(path) as src:
raw = src.read()
return json.loads(raw)
else:
return json.load(open(path))
def resolve_folder_title(uid):
return FOLDER_TITLES.get(uid, uid.upper())
def enrich_dashboard(dashboard, folder=None):
if not folder:
return dashboard
extra_tags = FOLDER_TAGS.get(folder, [])
if not extra_tags:
return dashboard
existing_tags = dashboard.get("tags", [])
if not isinstance(existing_tags, list):
existing_tags = []
merged_tags = []
seen = set()
for tag in existing_tags + extra_tags:
if not tag or tag in seen:
continue
seen.add(tag)
merged_tags.append(tag)
dashboard["tags"] = merged_tags
return dashboard
# json serializer: use compact_json if available, fallback to standard json
try:
from compact_json import Formatter
_formatter = Formatter()
_formatter.max_inline_length = 400
_formatter.max_inline_complexity = 10
_formatter.max_compact_list_complexity = 10
_formatter.indent_spaces = 2
_formatter.nested_bracket_padding = False
_formatter.simple_bracket_padding = False
_formatter.colon_padding = False
_formatter.comma_padding = False
def dump_json(data, file):
file.write(_formatter.serialize(data))
except:
def dump_json(data, file):
json.dump(data, file, indent=1, separators=(',', ':'), sort_keys=True)
def dump_dashboard_to_file(d, path):
with open(path, 'w') as dst:
raw = dashboard_raw(d)
raw["version"] = 1
raw["author"] = "Ruohang Feng (rh@vonng.com)"
raw["license"] = "https://pigsty.io/docs/about/license/"
dump_json(raw, dst)
#json.dump(raw, dst, indent=1, separators=(',', ':'), sort_keys=True)
def dump_dashboard(d, home):
db_uid = d["dashboard"]["uid"]
dir_uid = d["meta"].get("folderUid")
if dir_uid is None or dir_uid == "":
dir_uid = '.'
p = os.path.join(home, dir_uid, db_uid + '.json')
with open(p, 'w') as dst:
dump_json(dashboard_raw(d), dst)
##########################################
# business logic
##########################################
def init_all(dashboard_dir):
"""init grafana with dashboards dir content
similar to load_all, but will replace domain name placeholder
"""
update_orgname(1, 'Pigsty') # update default org name
# add_folder("pgsql", "PGSQL") # create dashboard folders
# add_folder("pgcat", "PGCAT")
# add_folder("pglog", "PGLOG")
# load home dashboards
folders = []
for f in os.listdir(dashboard_dir):
abs_path = os.path.join(dashboard_dir, f)
if os.path.isfile(abs_path) and f.endswith('.json') and not f.startswith('.'):
print("init dashboard : %s" % f)
add_dashboard(load_dashboard(abs_path, True))
if os.path.isdir(abs_path) and not SKIP_SUBFOLDERS:
folders.append((f, abs_path)) # folder name, abs path
home_uid = "home"
if home_uid:
star_dashboard_by_uid(home_uid) # home dashboards will be loaded above if exists
update_org_preference(home_uid, "light")
update_user_preference(home_uid, "light")
# load other second-layer dashboards
for folder_name, folder_path in folders:
print("init folder %s" % folder_name)
add_folder(folder_name, resolve_folder_title(folder_name))
for f in os.listdir(folder_path):
abs_path = os.path.join(dashboard_dir, folder_name, f)
if os.path.isfile(abs_path) and f.endswith('.json') and not f.startswith('.'):
print("init dashboard: %s / %s" % (folder_name, f))
add_dashboard(enrich_dashboard(load_dashboard(abs_path, True), folder_name), folder_name)
def load_all(dashboard_dir):
"""load dashboards and folders"""
folders = []
for f in os.listdir(dashboard_dir):
abs_path = os.path.join(dashboard_dir, f)
if os.path.isfile(abs_path) and f.endswith('.json') and not f.startswith('.'):
print("load dashboard : %s" % f)
add_dashboard(load_dashboard(abs_path))
if os.path.isdir(abs_path) and not SKIP_SUBFOLDERS:
folders.append((f, abs_path)) # folder name, abs path
for folder_name, folder_path in folders:
print("add folder %s" % folder_name)
add_folder(folder_name, resolve_folder_title(folder_name))
for f in os.listdir(folder_path):
abs_path = os.path.join(dashboard_dir, folder_name, f)
if os.path.isfile(abs_path) and f.endswith('.json') and not f.startswith('.'):
print("load dashboard: %s / %s" % (folder_name, f))
add_dashboard(enrich_dashboard(load_dashboard(abs_path), folder_name), folder_name)
def dump_all(dashboard_dir):
"""dump dashboard to specific dir with fhs"""
if not os.path.exists(dashboard_dir):
print("dump: create dashboard dir %s" % dashboard_dir)
os.mkdir(dashboard_dir)
dbmeta = list_dashboards()
folders = set([i.get('folderUid') for i in dbmeta if 'folderUid' in i and i.get('type') != 'dash-folder'])
dashdbs = [(i.get('uid'), i.get('folderUid', '.')) for i in dbmeta if i.get('type') != 'dash-folder']
for d in folders:
abs_path = os.path.join(dashboard_dir, d)
if os.path.isfile(abs_path):
os.remove(abs_path)
if not os.path.exists(abs_path):
print("dump: %s / dir created" % d)
os.mkdir(abs_path)
print("dump: create dir %s" % d)
for uid, folder in dashdbs:
dbpath = os.path.join(dashboard_dir, folder, uid + '.json')
if folder == "." or not folder:
dbpath = os.path.join(dashboard_dir, uid + '.json')
print("dump: %s / %s \t ---> %s" % (folder, uid, dbpath))
dump_dashboard_to_file(get_dashboard(uid), dbpath)
def clean_all():
dbmeta = list_dashboards()
folders = set([i.get('folderUid') for i in dbmeta if 'folderUid' in i and i.get('type') != 'dash-folder'])
dashdbs = [(i.get('uid'), i.get('folderUid', '.')) for i in dbmeta if
i.get('type') != 'dash-folder']
for d, f in dashdbs:
print("clean: dashboard %s" % d)
del_dashboard(d)
for f in folders:
print("clean: folder %s" % f)
del_folder(f)
def add_default_datasource():
for k, v in DEFAULT_DATASOURCES.items():
print("init: data source %s" % k)
create_datasource(v)
update_datasource(k, v)
def usage():
print("""
grafana.py [init|load|dump|clean]
init [dashboard_dir=.] # provisioning grafana
load [dashboard_dir=.] # load folders & dashboards
dump [dashboard_dir=.] # dump folders & dashboards
ds # init data sources
clean # clean folders & dashboards
""")
if __name__ == '__main__':
if len(sys.argv) <= 1:
usage()
exit(1)
action = sys.argv[1]
dashboard_dir_path = None
if len(sys.argv) > 2:
dashboard_dir_path = sys.argv[2]
else:
dashboard_dir_path = '.'
print("Grafana API: %s:%s @ %s" % (USERNAME, PASSWORD, ENDPOINT))
if action == 'clean':
print("clean all dashboards and folders")
clean_all()
exit(0)
if not (os.path.exists(dashboard_dir_path) and os.path.isdir(dashboard_dir_path)):
print("not exists : dashboard dir %s " % dashboard_dir_path)
exit(2)
if action == 'init':
# add_default_datasource()
init_all(dashboard_dir_path)
elif action == 'load':
load_all(dashboard_dir_path)
elif action == 'dump':
dump_all(dashboard_dir_path)
elif action == 'ds':
add_default_datasource()
else:
usage()
exit(3)
Reference in New Issue View Git Blame Copy Permalink