From 73d35a7add259261f67b2bfc780c9045c1523257 Mon Sep 17 00:00:00 2001 From: Haitao Pan Date: Fri, 22 Dec 2023 00:59:34 +0800 Subject: [PATCH] add apps/monitor: loki-stack, kube-prometheus-stack --- .../kube-state-metrics-config.yaml | 275 ++++++++++++++++++ .../kube-prometheus-stack/kustomization.yaml | 17 ++ .../kustomizeconfig.yaml | 6 + .../kube-prometheus-stack/namespace.yaml | 6 + .../kube-prometheus-stack/release.yaml | 54 ++++ .../kube-prometheus-stack/repository.yaml | 8 + apps/monitor/loki-stack/kustomization.yaml | 6 + apps/monitor/loki-stack/release.yaml | 34 +++ apps/monitor/loki-stack/repository.yaml | 7 + clusters/sit/kustomization.yaml | 2 +- clusters/sit/prometheus-agent.yaml | 44 --- 11 files changed, 414 insertions(+), 45 deletions(-) create mode 100644 apps/monitor/kube-prometheus-stack/kube-state-metrics-config.yaml create mode 100644 apps/monitor/kube-prometheus-stack/kustomization.yaml create mode 100644 apps/monitor/kube-prometheus-stack/kustomizeconfig.yaml create mode 100644 apps/monitor/kube-prometheus-stack/namespace.yaml create mode 100644 apps/monitor/kube-prometheus-stack/release.yaml create mode 100644 apps/monitor/kube-prometheus-stack/repository.yaml create mode 100644 apps/monitor/loki-stack/kustomization.yaml create mode 100644 apps/monitor/loki-stack/release.yaml create mode 100644 apps/monitor/loki-stack/repository.yaml delete mode 100644 clusters/sit/prometheus-agent.yaml diff --git a/apps/monitor/kube-prometheus-stack/kube-state-metrics-config.yaml b/apps/monitor/kube-prometheus-stack/kube-state-metrics-config.yaml new file mode 100644 index 0000000..e0694ad --- /dev/null +++ b/apps/monitor/kube-prometheus-stack/kube-state-metrics-config.yaml @@ -0,0 +1,275 @@ +kube-state-metrics: + # For kube-prometheus-stacks that are already installed and configured with + # custom collectors, commenting out the collectors and extraArgs below will + # retain any existing kube-state-metrics configuration. + collectors: [ ] + extraArgs: + - --custom-resource-state-only=true + rbac: + extraRules: + - apiGroups: + - source.toolkit.fluxcd.io + - kustomize.toolkit.fluxcd.io + - helm.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + - image.toolkit.fluxcd.io + resources: + - gitrepositories + - buckets + - helmrepositories + - helmcharts + - ocirepositories + - kustomizations + - helmreleases + - alerts + - providers + - receivers + - imagerepositories + - imagepolicies + - imageupdateautomations + verbs: [ "list", "watch" ] + customResourceState: + enabled: true + config: + spec: + resources: + - groupVersionKind: + group: kustomize.toolkit.fluxcd.io + version: v1 + kind: Kustomization + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, lastAppliedRevision ] + source_name: [ spec, sourceRef, name ] + - groupVersionKind: + group: helm.toolkit.fluxcd.io + version: v2beta2 + kind: HelmRelease + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, lastAppliedRevision ] + chart_name: [ spec, chart, spec, chart ] + chart_source_name: [ spec, chart, spec, sourceRef, name ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1 + kind: GitRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + url: [ spec, url ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: Bucket + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + endpoint: [ spec, endpoint ] + bucket_name: [ spec, bucketName ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: HelmRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + url: [ spec, url ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: HelmChart + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + chart_name: [ spec, chart ] + chart_version: [ spec, version ] + - groupVersionKind: + group: source.toolkit.fluxcd.io + version: v1beta2 + kind: OCIRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + revision: [ status, artifact, revision ] + url: [ spec, url ] + - groupVersionKind: + group: notification.toolkit.fluxcd.io + version: v1beta3 + kind: Alert + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + suspended: [ spec, suspend ] + - groupVersionKind: + group: notification.toolkit.fluxcd.io + version: v1beta3 + kind: Provider + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + suspended: [ spec, suspend ] + - groupVersionKind: + group: notification.toolkit.fluxcd.io + version: v1 + kind: Receiver + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + webhook_path: [ status, webhookPath ] + - groupVersionKind: + group: image.toolkit.fluxcd.io + version: v1beta2 + kind: ImageRepository + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + image: [ spec, image ] + - groupVersionKind: + group: image.toolkit.fluxcd.io + version: v1beta2 + kind: ImagePolicy + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + source_name: [ spec, imageRepositoryRef, name ] + - groupVersionKind: + group: image.toolkit.fluxcd.io + version: v1beta1 + kind: ImageUpdateAutomation + metricNamePrefix: gotk + metrics: + - name: "resource_info" + help: "The current state of a GitOps Toolkit resource." + each: + type: Info + info: + labelsFromPath: + name: [ metadata, name ] + labelsFromPath: + exported_namespace: [ metadata, namespace ] + ready: [ status, conditions, "[type=Ready]", status ] + suspended: [ spec, suspend ] + source_name: [ spec, sourceRef, name ] diff --git a/apps/monitor/kube-prometheus-stack/kustomization.yaml b/apps/monitor/kube-prometheus-stack/kustomization.yaml new file mode 100644 index 0000000..d39eb9f --- /dev/null +++ b/apps/monitor/kube-prometheus-stack/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: monitoring +resources: + - namespace.yaml + - repository.yaml + - release.yaml +configMapGenerator: + - name: flux-kube-state-metrics-config + files: + - kube-state-metrics-config.yaml + options: + labels: + app.kubernetes.io/part-of: flux + app.kubernetes.io/component: monitoring +configurations: + - kustomizeconfig.yaml diff --git a/apps/monitor/kube-prometheus-stack/kustomizeconfig.yaml b/apps/monitor/kube-prometheus-stack/kustomizeconfig.yaml new file mode 100644 index 0000000..7c31b22 --- /dev/null +++ b/apps/monitor/kube-prometheus-stack/kustomizeconfig.yaml @@ -0,0 +1,6 @@ +nameReference: +- kind: ConfigMap + version: v1 + fieldSpecs: + - path: spec/valuesFrom/name + kind: HelmRelease diff --git a/apps/monitor/kube-prometheus-stack/namespace.yaml b/apps/monitor/kube-prometheus-stack/namespace.yaml new file mode 100644 index 0000000..78abcfc --- /dev/null +++ b/apps/monitor/kube-prometheus-stack/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + labels: + app.kubernetes.io/component: monitoring diff --git a/apps/monitor/kube-prometheus-stack/release.yaml b/apps/monitor/kube-prometheus-stack/release.yaml new file mode 100644 index 0000000..98dd262 --- /dev/null +++ b/apps/monitor/kube-prometheus-stack/release.yaml @@ -0,0 +1,54 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: kube-prometheus-stack +spec: + interval: 1h + chart: + spec: + version: "55.x" + chart: kube-prometheus-stack + sourceRef: + kind: HelmRepository + name: prometheus-community + interval: 1h + install: + crds: Create + upgrade: + crds: CreateReplace + driftDetection: + mode: enabled + ignore: + # Ignore "validated" annotation which is not inserted during install + - paths: [ "/metadata/annotations/prometheus-operator-validated" ] + target: + kind: PrometheusRule + valuesFrom: + - kind: ConfigMap + name: flux-kube-state-metrics-config + valuesKey: kube-state-metrics-config.yaml + # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml + values: + prometheus: + agentMode: false + prometheusSpec: + remoteWrite: + - name: remote_prometheus + url: 'https://prometheus.svc.ink/api/v1/write' + retention: 24h + resources: + requests: + cpu: 200m + memory: 200Mi + podMonitorNamespaceSelector: { } + podMonitorSelector: + matchLabels: + app.kubernetes.io/component: monitoring + defaultRules: + create: false + grafana: + enabled: false + prometheus-windows-exporter: + enabled: false + alertmanager: + enabled: false diff --git a/apps/monitor/kube-prometheus-stack/repository.yaml b/apps/monitor/kube-prometheus-stack/repository.yaml new file mode 100644 index 0000000..f14b884 --- /dev/null +++ b/apps/monitor/kube-prometheus-stack/repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: prometheus-community +spec: + interval: 12h + type: oci + url: oci://ghcr.io/prometheus-community/charts diff --git a/apps/monitor/loki-stack/kustomization.yaml b/apps/monitor/loki-stack/kustomization.yaml new file mode 100644 index 0000000..d6d4338 --- /dev/null +++ b/apps/monitor/loki-stack/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: monitoring +resources: + - repository.yaml + - release.yaml diff --git a/apps/monitor/loki-stack/release.yaml b/apps/monitor/loki-stack/release.yaml new file mode 100644 index 0000000..dbd30f5 --- /dev/null +++ b/apps/monitor/loki-stack/release.yaml @@ -0,0 +1,34 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: loki-stack +spec: + interval: 5m + dependsOn: + - name: kube-prometheus-stack + chart: + spec: + version: "2.x" + chart: loki-stack + sourceRef: + kind: HelmRepository + name: grafana-charts + interval: 60m + # https://github.com/grafana/helm-charts/blob/main/charts/loki-stack/values.yaml + # https://github.com/grafana/loki/blob/main/production/helm/loki/values.yaml + values: + promtail: + enabled: true + loki: + enabled: true + isDefault: false + serviceMonitor: + enabled: true + additionalLabels: + app.kubernetes.io/part-of: kube-prometheus-stack + config: + chunk_store_config: + max_look_back_period: 0s + table_manager: + retention_deletes_enabled: true + retention_period: 12h diff --git a/apps/monitor/loki-stack/repository.yaml b/apps/monitor/loki-stack/repository.yaml new file mode 100644 index 0000000..49f58cd --- /dev/null +++ b/apps/monitor/loki-stack/repository.yaml @@ -0,0 +1,7 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: grafana-charts +spec: + interval: 120m0s + url: https://grafana.github.io/helm-charts diff --git a/clusters/sit/kustomization.yaml b/clusters/sit/kustomization.yaml index 5fedb91..90659b4 100644 --- a/clusters/sit/kustomization.yaml +++ b/clusters/sit/kustomization.yaml @@ -3,5 +3,5 @@ kind: Kustomization resources: - namespace.yaml - observability-agent.yaml - - prometheus-agent.yaml + - ../../apps/monitor/kube-prometheus-stack/ - podmonitor.yaml diff --git a/clusters/sit/prometheus-agent.yaml b/clusters/sit/prometheus-agent.yaml deleted file mode 100644 index 47ce675..0000000 --- a/clusters/sit/prometheus-agent.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1beta2 -kind: HelmRepository -metadata: - name: prometheus-community - namespace: monitoring -spec: - interval: 1m - url: https://prometheus-community.github.io/helm-charts ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: prometheus-operator - namespace: monitoring -spec: - chart: - spec: - chart: kube-prometheus-stack - version: "55.5.0" - sourceRef: - kind: HelmRepository - name: prometheus-community - namespace: monitoring - interval: 1m - values: - prometheusOperator: - enabled: true - crds: - enabled: true - prometheus: - enabled: true - agentMode: false - prometheusSpec: - remoteWrite: - - name: remote_prometheus - url: 'https://prometheus.svc.ink/api/v1/write' - defaultRules: - create: false - grafana: - enabled: false - prometheus-windows-exporter: - enabled: false - alertmanager: - enabled: false