Provision Grafana folders with semantic file providers

files/grafana/grafana.py

@@ -10,11 +10,19 @@
 #==============================================================#
 import os, sys, json, requests
 
+
+def env_flag(name, default):
+    value = os.environ.get(name)
+    if value is None:
+        return default
+    return value.lower() in ('1', 'true', 'yes', 'on')
+
 # grafana access info
 ENDPOINT = os.environ.get("GRAFANA_ENDPOINT", 'http://i.pigsty/ui')
 USERNAME = os.environ.get("GRAFANA_USERNAME", 'admin')
 PASSWORD = os.environ.get("GRAFANA_PASSWORD", 'pigsty')
-CREATE_FOLDERS = True
+CREATE_FOLDERS = env_flag('GRAFANA_CREATE_FOLDERS', True)
+SKIP_SUBFOLDERS = env_flag('GRAFANA_SKIP_SUBFOLDERS', False)
 
 FOLDER_TITLES = {
     '01-iaas-compute': 'IAAS / 计算',
@@ -327,7 +335,7 @@ def init_all(dashboard_dir):
         if os.path.isfile(abs_path) and f.endswith('.json')  and not f.startswith('.'):
             print("init dashboard : %s" % f)
             add_dashboard(load_dashboard(abs_path, True))
-        if os.path.isdir(abs_path):
+        if os.path.isdir(abs_path) and not SKIP_SUBFOLDERS:
             folders.append((f, abs_path))  # folder name, abs path
 
     home_uid = "home"
@@ -356,7 +364,7 @@ def load_all(dashboard_dir):
         if os.path.isfile(abs_path) and f.endswith('.json') and not f.startswith('.'):
             print("load dashboard : %s" % f)
             add_dashboard(load_dashboard(abs_path))
-        if os.path.isdir(abs_path):
+        if os.path.isdir(abs_path) and not SKIP_SUBFOLDERS:
             folders.append((f, abs_path))  # folder name, abs path
 
     for folder_name, folder_path in folders:

roles/infra/tasks/grafana.yml

@@ -90,12 +90,36 @@
   when: grafana_enabled|bool
   block:
 
-    # sync files/grafana @ local -> /etc/dashboards @ infra
+    - name: remove legacy grafana dashboard directories
+      tags: dashboard_sync
+      file: path={{ item }} state=absent
+      with_items:
+        - /etc/dashboards/app
+        - /etc/dashboards/infra
+        - /etc/dashboards/minio
+        - /etc/dashboards/mongo
+        - /etc/dashboards/node
+        - /etc/dashboards/pgsql
+        - /etc/dashboards/redis
+        - /etc/dashboards/__pycache__
+        - /infra/dashboards/app
+        - /infra/dashboards/infra
+        - /infra/dashboards/minio
+        - /infra/dashboards/mongo
+        - /infra/dashboards/node
+        - /infra/dashboards/pgsql
+        - /infra/dashboards/redis
+        - /infra/dashboards/__pycache__
+
+    # sync files/grafana @ local -> /etc/dashboards + /infra/dashboards
     - name: sync grafana dashboards
       tags: dashboard_sync
-      copy: src=grafana/ dest=/infra/dashboards/
+      copy: src=grafana/ dest={{ item }}
+      with_items:
+        - /etc/dashboards/
+        - /infra/dashboards/
 
-    - name: provisioning grafana with grafana.py
+    - name: provision root dashboards with grafana.py
       when: inventory_hostname in groups["infra"]|default([])
       throttle: "{% if grafana_pgurl != '' %}1{% else %}5{% endif %}"
       tags: dashboard_init
@@ -109,8 +133,11 @@
         export GRAFANA_ENDPOINT={{ endpoint }}
         export GRAFANA_USERNAME={{ username }}
         export GRAFANA_PASSWORD={{ password }}
+        export GRAFANA_CREATE_FOLDERS=false
+        export GRAFANA_SKIP_SUBFOLDERS=true
 
-        # run provisioning logic
+        # keep API provisioning only for root dashboards such as homepage;
+        # folder dashboards are provisioned by Grafana file providers.
         cd /infra/dashboards/
         chown -R root:root /infra/dashboards/
         /bin/python3 /infra/dashboards/grafana.py init /infra/dashboards/

roles/infra/templates/grafana/dashboard.yml.j2

@@ -12,12 +12,102 @@
 apiVersion: 1
 providers:
 
-  - name: Pigsty
+  - name: IAAS Compute
     orgId: 1
     type: file
     updateIntervalSeconds: 8
+    disableDeletion: false
     allowUiUpdates: true
+    folder: IAAS / 计算
+    folderUid: 01-iaas-compute
     options:
-      path: /etc/dashboards/
-      foldersFromFilesStructure: true
+      path: /etc/dashboards/01-iaas-compute
+
+  - name: IAAS Storage
+    orgId: 1
+    type: file
+    updateIntervalSeconds: 8
+    disableDeletion: false
+    allowUiUpdates: true
+    folder: IAAS / 存储
+    folderUid: 02-iaas-storage
+    options:
+      path: /etc/dashboards/02-iaas-storage
+
+  - name: IAAS Network
+    orgId: 1
+    type: file
+    updateIntervalSeconds: 8
+    disableDeletion: false
+    allowUiUpdates: true
+    folder: IAAS / 网络
+    folderUid: 03-iaas-network
+    options:
+      path: /etc/dashboards/03-iaas-network
+
+  - name: PAAS Control Plane
+    orgId: 1
+    type: file
+    updateIntervalSeconds: 8
+    disableDeletion: false
+    allowUiUpdates: true
+    folder: PaaS / 平台控制面
+    folderUid: 11-paas-control-plane
+    options:
+      path: /etc/dashboards/11-paas-control-plane
+
+  - name: PAAS Cluster
+    orgId: 1
+    type: file
+    updateIntervalSeconds: 8
+    disableDeletion: false
+    allowUiUpdates: true
+    folder: PaaS / 集群
+    folderUid: 12-paas-cluster
+    options:
+      path: /etc/dashboards/12-paas-cluster
+
+  - name: PAAS DB
+    orgId: 1
+    type: file
+    updateIntervalSeconds: 8
+    disableDeletion: false
+    allowUiUpdates: true
+    folder: PaaS / DB
+    folderUid: 13-paas-db
+    options:
+      path: /etc/dashboards/13-paas-db
+
+  - name: PAAS Cache
+    orgId: 1
+    type: file
+    updateIntervalSeconds: 8
+    disableDeletion: false
+    allowUiUpdates: true
+    folder: PaaS / 缓存
+    folderUid: 14-paas-cache
+    options:
+      path: /etc/dashboards/14-paas-cache
+
+  - name: BU Proxy
+    orgId: 1
+    type: file
+    updateIntervalSeconds: 8
+    disableDeletion: false
+    allowUiUpdates: true
+    folder: 业务单元 / 代理
+    folderUid: 22-bu-proxy
+    options:
+      path: /etc/dashboards/22-bu-proxy
+
+  - name: BU Request
+    orgId: 1
+    type: file
+    updateIntervalSeconds: 8
+    disableDeletion: false
+    allowUiUpdates: true
+    folder: 业务单元 / 请求
+    folderUid: 24-bu-request
+    options:
+      path: /etc/dashboards/24-bu-request
 ...