diff --git a/clusters/sit/podmonitor.yaml b/apps/monitor/podmonitor.yaml similarity index 100% rename from clusters/sit/podmonitor.yaml rename to apps/monitor/podmonitor.yaml diff --git a/clusters/sit/prometheus-agent.yaml b/apps/monitor/prometheus-agent.yaml similarity index 100% rename from clusters/sit/prometheus-agent.yaml rename to apps/monitor/prometheus-agent.yaml diff --git a/clusters/sit/fluxcd-metrics.yaml b/clusters/sit/fluxcd-metrics.yaml new file mode 100644 index 0000000..f678267 --- /dev/null +++ b/clusters/sit/fluxcd-metrics.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: kustomize-controller-metrics + labels: + app: kustomize-controller +spec: + selector: + app: kustomize-controller + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: helm-controller-metrics + labels: + app: helm-controller +spec: + selector: + app: helm-controller + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: source-controller-metrics + labels: + app: source-controller +spec: + selector: + app: source-controller + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/clusters/sit/kustomization.yaml b/clusters/sit/kustomization.yaml index b2e73b1..67396e8 100644 --- a/clusters/sit/kustomization.yaml +++ b/clusters/sit/kustomization.yaml @@ -4,3 +4,7 @@ namespace: monitoring resources: - namespace.yaml - observability-agent.yaml + - prometheus-agent-configmap.yaml + - fluxcd-metrics.yaml +patchesStrategicMerge: + - prometheus-agent-configmap-patch-fluxcd.yaml diff --git a/clusters/sit/prometheus-agent-configmap-patch-fluxcd.yaml b/clusters/sit/prometheus-agent-configmap-patch-fluxcd.yaml new file mode 100644 index 0000000..8824677 --- /dev/null +++ b/clusters/sit/prometheus-agent-configmap-patch-fluxcd.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +data: + prometheus.yml: | + scrape_configs: + - job_name: 'fluxcd-controller-metrics' + targets: + - 'kustomize-controller-metrics:8080' + - 'helm-controller-metrics:8080' + - 'source-controller-metrics:8080' +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: observabilityagent + meta.helm.sh/release-namespace: monitoring + labels: + app: prometheus + app.kubernetes.io/managed-by: Helm + chart: prometheus-18.4.0 + component: server + helm.toolkit.fluxcd.io/name: observabilityagent + helm.toolkit.fluxcd.io/namespace: monitoring + heritage: Helm + release: observabilityagent + name: observabilityagent-prometheus-server + namespace: monitoring diff --git a/clusters/sit/prometheus-agent-configmap.yaml b/clusters/sit/prometheus-agent-configmap.yaml new file mode 100644 index 0000000..d4e6099 --- /dev/null +++ b/clusters/sit/prometheus-agent-configmap.yaml @@ -0,0 +1,318 @@ +apiVersion: v1 +data: + alerting_rules.yml: | + {} + alerts: | + {} + allow-snippet-annotations: "false" + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + remote_write: + - name: remote_prometheus + url: https://prometheus.svc.ink/api/v1/write + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - /etc/config/rules + - /etc/config/alerts + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + scrape_interval: 5m + scrape_timeout: 30s + recording_rules.yml: | + {} + rules: | + {} +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: observabilityagent + meta.helm.sh/release-namespace: monitoring + labels: + app: prometheus + app.kubernetes.io/managed-by: Helm + chart: prometheus-18.4.0 + component: server + helm.toolkit.fluxcd.io/name: observabilityagent + helm.toolkit.fluxcd.io/namespace: monitoring + heritage: Helm + release: observabilityagent + name: observabilityagent-prometheus-server + namespace: monitoring