observability-server: add role and ci pipeline

.github/workflows/pulumi-cloud-resource-observability-server.yml

@@ -0,0 +1,69 @@
+name: setup observability-server 
+
+on:
+  workflow_dispatch:
+    branches: [ 'main' ]
+
+env:
+  DNS_AK: ${{ secrets.DNS_AK }}
+  DNS_SK: ${{ secrets.DNS_SK }}
+  SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }}
+  SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+  OIDC_ADMIN_PASSWORD: ${{ OIDC_ADMIN_PASSWORD }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
+  PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_PASS }}
+  PULUMI_DISABLE_CI_DETECTION: true
+  AWS_REGION: ap-east-1
+
+defaults:
+  run:
+    working-directory: ./playbook
+
+jobs:
+  observability-server:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: run playbook
+        shell: bash
+        run: |
+          export ANSIBLE_HOST_KEY_CHECKING=False
+
+          sudo apt install jq ansible -y
+          sudo rm -rvf /home/runner/.pulumi/
+          curl -o setup.sh https://get.pulumi.com && sh setup.sh --version 3.57.0
+
+          pulumi login s3://pulumi-aws-dev
+          pulumi stack select dev
+
+          db_server_public_ip=`pulumi stack output --json | jq '.db_server_public_ip'`
+          k3s_server_public_ip=`pulumi stack output --json | jq '.k3s_server_public_ip'`
+          
+          mkdir -pv ~/.ssh/
+          cat > ~/.ssh/id_rsa << EOF
+          ${SSH_PRIVATE_KEY}
+          EOF
+          sudo chmod 0400 ~/.ssh/id_rsa
+          md5sum ~/.ssh/id_rsa
+          
+          cat > hosts/inventory << EOF
+          [master]
+          k3s-server               ansible_host=$k3s_server_public_ip
+          
+          [node]
+          db-server                ansible_host=$db_server_public_ip
+          
+          [all:vars]
+          ansible_port=22
+          ansible_ssh_user=ubuntu
+          ansible_ssh_private_key_file=~/.ssh/id_rsa
+          ansible_host_key_checking=False
+          dns_ak=$DNS_AK
+          dns_sk=$DNS_SK
+          keycloak_ui_password=$OIDC_ADMIN_PASSWORD
+          lb_ip=$k3s_server_public_ip
+          EOF
+          ansible-playbook -i hosts/inventory jobs/init_observability-server -D

playbook/jobs/init_observability-server

@@ -0,0 +1,13 @@
+- name: setup observability server 
+  hosts: all
+  user: root
+  become: yes
+  gather_facts: yes
+  tasks:
+    - include_role:
+        name: observability-server
+      vars:
+        group: master
+        namespace: monitoring
+        domain: onwalk.net
+        secret: obs-tls

playbook/roles/observability-server/files/setup-observable-server.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+export domain=$1
+export secret=$2
+export namespace=$3
+export mysql_db_password=$4
+
+kubectl label nodes k3s-server prometheus=true --overwrite
+
+cat > values.yaml << EOF
+deepflow:
+  enabled: true
+  clickhouse:
+    enabled: false
+  mysql:
+    enabled: false
+  grafana:
+    enabled: true
+    ingress:
+      enabled: true
+      ingressClassName: nginx
+      hosts:
+        - grafana.${domain}
+      tls:
+        - secretName: ${secret}
+          hosts:
+            - grafana.${domain}
+  global:
+    externalClickHouse:
+      enabled: true
+      type: ep
+      clusterName: default
+      storagePolicy: default
+      username: default
+      password: ''
+      hosts:
+      - ip: 10.1.2.3
+        port: 9000
+      - ip: 10.1.2.4
+        port: 9000
+      - ip: 10.1.2.5
+        port: 9000
+    externalMySQL:
+      enabled: true
+      ip: mysql.database.svc.cluster.local
+      port: 3306
+      username: root
+      password: {{ mysql_db_password }}
+prometheus:
+  enabled: true
+  alertmanager:
+    enabled: false
+  prometheus-pushgateway:
+    enabled: false
+  kube-state-metrics:
+    enabled: false
+  server:
+    ingress:
+      ingressClassName: nginx
+      hosts:
+        - prometheus.${domain}
+      tls:
+        - secretName: ${secret}
+          hosts:
+            - prometheus.${domain}
+    alertmanagers:
+    - static_configs:
+      - targets:
+        - alertmanager.${domain}
+  serverFiles:
+    prometheus.yml:
+      rule_files:
+        - /etc/config/recording_rules.yml
+        - /etc/config/alerting_rules.yml
+alertmanager:
+  configmapReload:
+    enabled: false
+  config:
+    global:
+      resolve_timeout: 5m
+      smtp_smarthost: 'smtp.qq.com:465'
+      smtp_from: '11111111@qq.com'
+      smtp_auth_username: '11111111@qq.com'
+      smtp_auth_password: '123456'
+      smtp_require_tls: false
+    templates:
+      - '/etc/alertmanager/*.tmpl'
+    receivers:
+      - name: 'default-receiver'
+        email_configs:
+        - to: '{{ template "email.to" . }}'
+          html: '{{ template "email.to.html" . }}'
+    route:
+      group_wait: 10s
+      group_interval: 5m
+      receiver: default-receiver
+      repeat_interval: 1h
+EOF
+
+helm repo add stable https://artifact.onwalk.net/chartrepo/public/ || echo true
+helm repo update
+helm upgrade --install observable-server stable/observableserver -n ${namspace} -f values.yaml

playbook/roles/observability-server/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - role: cert-manager

playbook/roles/observability-server/tasks/main.yml

@@ -0,0 +1,13 @@
+- name: get db password
+  shell: 'kubectl get secret --namespace database postgresql -o jsonpath="{.data.postgres-password}" | base64 -d'
+  register: command_raw
+  when: inventory_hostname in groups[group][0]
+
+- name: set fact join command
+  set_fact:
+    mysql_db_password : "{{ command_raw.stdout_lines[0] }}"
+  when: inventory_hostname in groups[group][0]
+
+- name: Setup OpenLdap Server
+  script: files/setup-observable-server.sh {{ domain }} {{ secret }} {{ namespace }} {{ mysql_db_password }}
+  when: inventory_hostname in groups[group]