diff --git a/.github/workflows/pulumi-cloud-resource-observability-server.yml b/.github/workflows/pulumi-cloud-resource-observability-server.yml new file mode 100644 index 00000000..e65efa53 --- /dev/null +++ b/.github/workflows/pulumi-cloud-resource-observability-server.yml @@ -0,0 +1,69 @@ +name: setup observability-server + +on: + workflow_dispatch: + branches: [ 'main' ] + +env: + DNS_AK: ${{ secrets.DNS_AK }} + DNS_SK: ${{ secrets.DNS_SK }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + OIDC_ADMIN_PASSWORD: ${{ OIDC_ADMIN_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} + PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_PASS }} + PULUMI_DISABLE_CI_DETECTION: true + AWS_REGION: ap-east-1 + +defaults: + run: + working-directory: ./playbook + +jobs: + observability-server: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: run playbook + shell: bash + run: | + export ANSIBLE_HOST_KEY_CHECKING=False + + sudo apt install jq ansible -y + sudo rm -rvf /home/runner/.pulumi/ + curl -o setup.sh https://get.pulumi.com && sh setup.sh --version 3.57.0 + + pulumi login s3://pulumi-aws-dev + pulumi stack select dev + + db_server_public_ip=`pulumi stack output --json | jq '.db_server_public_ip'` + k3s_server_public_ip=`pulumi stack output --json | jq '.k3s_server_public_ip'` + + mkdir -pv ~/.ssh/ + cat > ~/.ssh/id_rsa << EOF + ${SSH_PRIVATE_KEY} + EOF + sudo chmod 0400 ~/.ssh/id_rsa + md5sum ~/.ssh/id_rsa + + cat > hosts/inventory << EOF + [master] + k3s-server ansible_host=$k3s_server_public_ip + + [node] + db-server ansible_host=$db_server_public_ip + + [all:vars] + ansible_port=22 + ansible_ssh_user=ubuntu + ansible_ssh_private_key_file=~/.ssh/id_rsa + ansible_host_key_checking=False + dns_ak=$DNS_AK + dns_sk=$DNS_SK + keycloak_ui_password=$OIDC_ADMIN_PASSWORD + lb_ip=$k3s_server_public_ip + EOF + ansible-playbook -i hosts/inventory jobs/init_observability-server -D diff --git a/playbook/jobs/init_observability-server b/playbook/jobs/init_observability-server new file mode 100644 index 00000000..2a096854 --- /dev/null +++ b/playbook/jobs/init_observability-server @@ -0,0 +1,13 @@ +- name: setup observability server + hosts: all + user: root + become: yes + gather_facts: yes + tasks: + - include_role: + name: observability-server + vars: + group: master + namespace: monitoring + domain: onwalk.net + secret: obs-tls diff --git a/playbook/roles/observability-server/files/setup-observable-server.sh b/playbook/roles/observability-server/files/setup-observable-server.sh new file mode 100644 index 00000000..848ebe09 --- /dev/null +++ b/playbook/roles/observability-server/files/setup-observable-server.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +export domain=$1 +export secret=$2 +export namespace=$3 +export mysql_db_password=$4 + +kubectl label nodes k3s-server prometheus=true --overwrite + +cat > values.yaml << EOF +deepflow: + enabled: true + clickhouse: + enabled: false + mysql: + enabled: false + grafana: + enabled: true + ingress: + enabled: true + ingressClassName: nginx + hosts: + - grafana.${domain} + tls: + - secretName: ${secret} + hosts: + - grafana.${domain} + global: + externalClickHouse: + enabled: true + type: ep + clusterName: default + storagePolicy: default + username: default + password: '' + hosts: + - ip: 10.1.2.3 + port: 9000 + - ip: 10.1.2.4 + port: 9000 + - ip: 10.1.2.5 + port: 9000 + externalMySQL: + enabled: true + ip: mysql.database.svc.cluster.local + port: 3306 + username: root + password: {{ mysql_db_password }} +prometheus: + enabled: true + alertmanager: + enabled: false + prometheus-pushgateway: + enabled: false + kube-state-metrics: + enabled: false + server: + ingress: + ingressClassName: nginx + hosts: + - prometheus.${domain} + tls: + - secretName: ${secret} + hosts: + - prometheus.${domain} + alertmanagers: + - static_configs: + - targets: + - alertmanager.${domain} + serverFiles: + prometheus.yml: + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml +alertmanager: + configmapReload: + enabled: false + config: + global: + resolve_timeout: 5m + smtp_smarthost: 'smtp.qq.com:465' + smtp_from: '11111111@qq.com' + smtp_auth_username: '11111111@qq.com' + smtp_auth_password: '123456' + smtp_require_tls: false + templates: + - '/etc/alertmanager/*.tmpl' + receivers: + - name: 'default-receiver' + email_configs: + - to: '{{ template "email.to" . }}' + html: '{{ template "email.to.html" . }}' + route: + group_wait: 10s + group_interval: 5m + receiver: default-receiver + repeat_interval: 1h +EOF + +helm repo add stable https://artifact.onwalk.net/chartrepo/public/ || echo true +helm repo update +helm upgrade --install observable-server stable/observableserver -n ${namspace} -f values.yaml diff --git a/playbook/roles/observability-server/meta/main.yml b/playbook/roles/observability-server/meta/main.yml new file mode 100644 index 00000000..83cef7b5 --- /dev/null +++ b/playbook/roles/observability-server/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - role: cert-manager diff --git a/playbook/roles/observability-server/tasks/main.yml b/playbook/roles/observability-server/tasks/main.yml new file mode 100755 index 00000000..66445bce --- /dev/null +++ b/playbook/roles/observability-server/tasks/main.yml @@ -0,0 +1,13 @@ +- name: get db password + shell: 'kubectl get secret --namespace database postgresql -o jsonpath="{.data.postgres-password}" | base64 -d' + register: command_raw + when: inventory_hostname in groups[group][0] + +- name: set fact join command + set_fact: + mysql_db_password : "{{ command_raw.stdout_lines[0] }}" + when: inventory_hostname in groups[group][0] + +- name: Setup OpenLdap Server + script: files/setup-observable-server.sh {{ domain }} {{ secret }} {{ namespace }} {{ mysql_db_password }} + when: inventory_hostname in groups[group]