Initial implementation (#1)

* Update `LICENSE` and `.gitignore`

* Init

* Update `README`

* Add Helm chart

* Update `README`

* Update chart

* Add `requirements.yaml`

* Update chart

* Update chart

* Update chart

* Update chart

* Update `Go` code

* Update `Go` code

* Update `Go` code

* Update `Go` code

* Update `README`

* Update `README`

* Update `Go` files

* Update `Go` files

* Update `Go` files

* Add Helm chart

* Update chart

* Update `README`

* Update `Go` files

* Update charts

* Update charts

* If AWS credentials are not provided in the command-line arguments or ENV variables, the module will try to assume an IAM Role

* Rename folder

* Update `README`

* Update `README`

* Rename folder

* Update `README`

* Add `requirements.yaml`

* Update chart
This commit is contained in:
Andriy Knysh 2018-03-21 16:19:55 -04:00 committed by GitHub
parent daebddabe3
commit f0005fccec
22 changed files with 915 additions and 1 deletions

7
.gitignore vendored
View File

@ -12,3 +12,10 @@
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/
# JetBrains
.idea
*.iml
.build-harness
dist/bin/*

45
.travis.yml Normal file
View File

@ -0,0 +1,45 @@
sudo: required
language: go
go:
- 1.9.x
addons:
apt:
packages:
- git
- make
- curl
env:
- DOCKER_IMAGE_NAME=cloudposse/prometheus-to-cloudwatch
services:
- docker
install:
- make init
- make travis/docker-login
- make go/deps-build
- make go/deps-dev
- make go-get
script:
- make go/deps
- make go/test
- make go/lint
- make go/build-all
- ls -l release/
- make docker/build
after_success:
- make travis/docker-tag-and-push
deploy:
- provider: releases
api_key: "$GITHUB_API_KEY"
file_glob: true
file: "release/*"
skip_cleanup: true
on:
tags: true

12
Dockerfile Normal file
View File

@ -0,0 +1,12 @@
FROM golang:1.10.0 as builder
RUN mkdir -p /go/src/github.com/cloudposse/prometheus-to-cloudwatch
WORKDIR /go/src/github.com/cloudposse/prometheus-to-cloudwatch
COPY . .
RUN go get && CGO_ENABLED=0 go build -v -o "./dist/bin/prometheus-to-cloudwatch" *.go
FROM alpine:3.7
RUN apk add --no-cache ca-certificates
COPY --from=builder /go/src/github.com/cloudposse/prometheus-to-cloudwatch/dist/bin/prometheus-to-cloudwatch /usr/bin/prometheus-to-cloudwatch
ENV PATH $PATH:/usr/bin
ENTRYPOINT ["prometheus-to-cloudwatch"]

View File

@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Copyright 2018 Cloud Posse, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

15
Makefile Normal file
View File

@ -0,0 +1,15 @@
SHELL = /bin/bash
PATH:=$(PATH):$(GOPATH)/bin
include $(shell curl --silent -o .build-harness "https://raw.githubusercontent.com/cloudposse/build-harness/master/templates/Makefile.build-harness"; echo .build-harness)
.PHONY : go-get
go-get:
go get
.PHONY : go-build
go-build: go-get
CGO_ENABLED=0 go build -v -o "./dist/bin/prometheus-to-cloudwatch" *.go

212
README.md Normal file
View File

@ -0,0 +1,212 @@
# prometheus-to-cloudwatch [![Build Status](https://travis-ci.org/cloudposse/prometheus-to-cloudwatch.svg?branch=master)](https://travis-ci.org/cloudposse/prometheus-to-cloudwatch)
Utility for scraping Prometheus metrics from a Prometheus client endpoint and publishing them to CloudWatch
## Usage
__NOTE__: The module accepts parameters as command-line arguments or as ENV variables (or any combination of command-line arguments and ENV vars).
Command-line arguments take precedence over ENV vars
| Command-line argument | ENV var | Description |
|:-----------------------------|:-----------------------------|:------------------------------------------------------------------------------|
| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics |
| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics |
| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace |
| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region |
| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds |
| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds |
| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from |
| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) |
| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) |
| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing |
__NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`)
or ENV variables (`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`),
the chain of credential providers will search for credentials in the shared credential file and EC2 Instance Roles.
This is useful when deploying the module in AWS on Kubernetes with [`kube2iam`](https://github.com/jtblin/kube2iam),
which will provide IAM credentials to containers running inside a Kubernetes cluster, allowing the module to assume an IAM Role with permissions
to publish metrics to CloudWatch.
## Examples
### Build Go program
```sh
go get
CGO_ENABLED=0 go build -v -o "./dist/bin/prometheus-to-cloudwatch" *.go
```
### Run locally
```sh
export AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXXXXXXXXXXX
export AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
export CLOUDWATCH_NAMESPACE=kube-state-metrics
export CLOUDWATCH_REGION=us-east-1
export CLOUDWATCH_PUBLISH_TIMEOUT=5
export PROMETHEUS_SCRAPE_INTERVAL=30
export PROMETHEUS_SCRAPE_URL=http://xxxxxxxxxxxx:8080/metrics
export PROMETHEUS_CERT_PATH=""
export PROMETHEUS_KEY_PATH=""
export PROMETHEUS_ACCEPT_INVALID_CERT=true
./dist/bin/prometheus-to-cloudwatch
```
### Build Docker image
__NOTE__: it will download all `Go` dependencies and then build the program inside the container (see [`Dockerfile`](Dockerfile))
```sh
docker build --tag prometheus-to-cloudwatch --no-cache=true .
```
### Run in a Docker container
```sh
docker run -i --rm \
-e AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXXXXXXXXXXX \
-e AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \
-e CLOUDWATCH_NAMESPACE=kube-state-metrics \
-e CLOUDWATCH_REGION=us-east-1 \
-e CLOUDWATCH_PUBLISH_TIMEOUT=5 \
-e PROMETHEUS_SCRAPE_INTERVAL=30 \
-e PROMETHEUS_SCRAPE_URL=http://xxxxxxxxxxxx:8080/metrics \
-e PROMETHEUS_CERT_PATH="" \
-e PROMETHEUS_KEY_PATH="" \
-e PROMETHEUS_ACCEPT_INVALID_CERT=true \
prometheus-to-cloudwatch
```
### Run on Kubernetes
To run on `Kubernetes`, we will deploy two [`Helm`](https://helm.sh/) [charts](https://docs.helm.sh/developing_charts/)
1. [kube-state-metrics](https://github.com/kubernetes/charts/tree/master/stable/kube-state-metrics) - to generates metrics about the state of various objects inside the cluster, such as deployments, nodes and pods
2. [prometheus-to-cloudwatch](chart) - to scrape metrics from `kube-state-metrics` and publish them to CloudWatch
Install `kube-state-metrics` chart
```sh
helm install stable/kube-state-metrics
```
Find the running services
```sh
kubectl get services
```
![kube-state-metrics-service](images/kube-state-metrics-service.png)
Copy the name of the `kube-state-metrics` service (`gauche-turtle-kube-state-metrics`) into the ENV var `PROMETHEUS_SCRAPE_URL` in [values.yaml](chart/values.yaml).
It should look like this:
```sh
PROMETHEUS_SCRAPE_URL: "http://gauche-turtle-kube-state-metrics:8080/metrics"
```
Deploy `prometheus-to-cloudwatch` chart
```sh
cd chart
helm install .
```
`prometheus-to-cloudwatch` will start scraping the `/metrics` endpoint of the `kube-state-metrics` service and send the Prometheus metrics to CloudWatch
![kube-state-metrics-to-cloudwatch](images/kube-state-metrics-to-cloudwatch.png)
## Help
**Got a question?**
File a GitHub [issue](https://github.com/cloudposse/prometheus-to-cloudwatch/issues), send us an [email](mailto:hello@cloudposse.com) or reach out to us on [Gitter](https://gitter.im/cloudposse/).
## Contributing
### Bug Reports & Feature Requests
Please use the [issue tracker](https://github.com/cloudposse/prometheus-to-cloudwatch/issues) to report any bugs or file feature requests.
### Developing
If you are interested in being a contributor and want to get involved in developing `prometheus-to-cloudwatch`, we would love to hear from you! Shoot us an [email](mailto:hello@cloudposse.com).
In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow.
1. **Fork** the repo on GitHub
2. **Clone** the project to your own machine
3. **Commit** changes to your own branch
4. **Push** your work back up to your fork
5. Submit a **Pull request** so that we can review your changes
**NOTE:** Be sure to merge the latest from "upstream" before making a pull request!
## License
[APACHE 2.0](LICENSE) © 2018 [Cloud Posse, LLC](https://cloudposse.com)
See [LICENSE](LICENSE) for full details.
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
## About
`prometheus-to-cloudwatch` is maintained and funded by [Cloud Posse, LLC][website].
![Cloud Posse](https://cloudposse.com/logo-300x69.png)
Like it? Please let us know at <hello@cloudposse.com>
We love [Open Source Software](https://github.com/cloudposse/)!
See [our other projects][community]
or [hire us][hire] to help build your next cloud platform.
[website]: https://cloudposse.com/
[community]: https://github.com/cloudposse/
[hire]: https://cloudposse.com/contact/
### Contributors
| [![Erik Osterman][erik_img]][erik_web]<br/>[Erik Osterman][erik_web] | [![Andriy Knysh][andriy_img]][andriy_web]<br/>[Andriy Knysh][andriy_web] |
|-------------------------------------------------------|------------------------------------------------------------------|
[erik_img]: http://s.gravatar.com/avatar/88c480d4f73b813904e00a5695a454cb?s=144
[erik_web]: https://github.com/osterman/
[andriy_img]: https://avatars0.githubusercontent.com/u/7356997?v=4&u=ed9ce1c9151d552d985bdf5546772e14ef7ab617&s=144
[andriy_web]: https://github.com/aknysh/

9
chart/.editorconfig Normal file
View File

@ -0,0 +1,9 @@
root = true
[*]
charset = utf-8
end_of_line = lf
indent_size = 2
indent_style = space
insert_final_newline = true
trim_trailing_whitespace = true

21
chart/.helmignore Normal file
View File

@ -0,0 +1,21 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj

5
chart/Chart.yaml Normal file
View File

@ -0,0 +1,5 @@
apiVersion: v1
appVersion: "1.0"
description: A Helm chart for prometheus-to-cloudwatch
name: prometheus-to-cloudwatch
version: 0.1.0

7
chart/README.md Normal file
View File

@ -0,0 +1,7 @@
# prometheus-to-cloudwatch
## Installation
```
helm install --name=prometheus-to-cloudwatch --namespace=prometheus-to-cloudwatch
```

Binary file not shown.

6
chart/requirements.lock Normal file
View File

@ -0,0 +1,6 @@
dependencies:
- name: common
repository: https://kubernetes-charts-incubator.storage.googleapis.com/
version: 0.0.1
digest: sha256:306378be27e855f6823d057c0d90ba3be227d44dc75b2c70c484a03c2515511e
generated: 2018-03-21T15:44:54.30424-04:00

4
chart/requirements.yaml Normal file
View File

@ -0,0 +1,4 @@
dependencies:
- name: common
version: 0.0.1
repository: https://kubernetes-charts-incubator.storage.googleapis.com/

20
chart/templates/NOTES.txt Normal file
View File

@ -0,0 +1,20 @@
Make sure to define the following settings in `values.yaml`:
env:
- CLOUDWATCH_NAMESPACE
- CLOUDWATCH_REGION
- CLOUDWATCH_PUBLISH_TIMEOUT
- PROMETHEUS_SCRAPE_INTERVAL
- PROMETHEUS_SCRAPE_URL
- PROMETHEUS_CERT_PATH
- PROMETHEUS_KEY_PATH
- PROMETHEUS_ACCEPT_INVALID_CERT
secrets:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
If you deploying the module in AWS on Kubernetes with `kube2iam` to assume an IAM Role with permissions
to publish metrics to CloudWatch, don't provide the AWS credentials in 'secrets'.
The chain of credential providers will search for credentials in EC2 Instance Roles.

View File

@ -0,0 +1,59 @@
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: {{ include "common.fullname" . }}
labels:
{{ include "common.labels.standard" . | indent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: {{ template "common.name" . }}
release: {{ .Release.Name | quote }}
strategy:
type: RollingUpdate
template:
metadata:
labels:
app: {{ template "common.name" . }}
release: {{ .Release.Name | quote }}
{{- if .Values.annotations }}
annotations:
{{ toYaml .Values.annotations | indent 8 }}
{{- end }}
spec:
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
{{- range $name, $value := .Values.env }}
{{- if not (empty $value) }}
- name: {{ $name | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
{{- $secret_name := include "common.fullname" . }}
{{- range $name, $value := .Values.secrets }}
{{- if not ( empty $value) }}
- name: {{ $name | quote }}
valueFrom:
secretKeyRef:
name: {{ $secret_name }}
key: {{ $name | quote }}
{{- end }}
{{- end }}
resources:
{{ toYaml .Values.resources | indent 12 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}

View File

@ -0,0 +1,13 @@
apiVersion: v1
kind: Secret
metadata:
name: {{ include "common.fullname" . }}
labels:
{{ include "common.labels.standard" . | indent 4 }}
type: Opaque
data:
{{- range $name, $value := .Values.secrets }}
{{- if not (empty $value) }}
{{ $name }}: {{ $value | b64enc }}
{{- end }}
{{- end }}

35
chart/values.yaml Normal file
View File

@ -0,0 +1,35 @@
# Default values for prometheus-to-cloudwatch
#
replicaCount: 1
env:
CLOUDWATCH_NAMESPACE: "kube-state-metrics"
CLOUDWATCH_REGION: "us-east-1"
CLOUDWATCH_PUBLISH_TIMEOUT: "5"
PROMETHEUS_SCRAPE_INTERVAL: "20"
PROMETHEUS_SCRAPE_URL: "http://xxxxxxxxxxxx:8080/metrics"
PROMETHEUS_CERT_PATH: ""
PROMETHEUS_KEY_PATH: ""
PROMETHEUS_ACCEPT_INVALID_CERT: "true"
secrets:
AWS_ACCESS_KEY_ID:
AWS_SECRET_ACCESS_KEY:
image:
repository: "cloudposse/prometheus-to-cloudwatch"
tag: "0.1.0"
pullPolicy: "IfNotPresent"
annotations: {}
resources: {}
nodeSelector: {}
tolerations: []
affinity: {}
volume:
emptyDir: {}

4
glide.yaml Normal file
View File

@ -0,0 +1,4 @@
package: github.com/cloudposse/prometheus-to-cloudwatch
import:
- package: github.com/aws/aws-sdk-go/aws
- package: github.com/prometheus

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 628 KiB

90
main.go Normal file
View File

@ -0,0 +1,90 @@
package main
import (
"flag"
"fmt"
"golang.org/x/net/context"
"log"
"os"
"strconv"
"time"
)
var (
awsAccessKeyId = flag.String("aws_access_key_id", os.Getenv("AWS_ACCESS_KEY_ID"), "AWS access key Id with permissions to publish CloudWatch metrics")
awsSecretAccessKey = flag.String("aws_secret_access_key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS secret access key with permissions to publish CloudWatch metrics")
cloudWatchNamespace = flag.String("cloudwatch_namespace", os.Getenv("CLOUDWATCH_NAMESPACE"), "CloudWatch Namespace")
cloudWatchRegion = flag.String("cloudwatch_region", os.Getenv("CLOUDWATCH_REGION"), "CloudWatch Region")
cloudWatchPublishTimeout = flag.String("cloudwatch_publish_timeout", os.Getenv("CLOUDWATCH_PUBLISH_TIMEOUT"), "CloudWatch publish timeout in seconds")
prometheusScrapeInterval = flag.String("prometheus_scrape_interval", os.Getenv("PROMETHEUS_SCRAPE_INTERVAL"), "Prometheus scrape interval in seconds")
prometheusScrapeUrl = flag.String("prometheus_scrape_url", os.Getenv("PROMETHEUS_SCRAPE_URL"), "Prometheus scrape URL")
certPath = flag.String("cert_path", os.Getenv("CERT_PATH"), "Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`)")
keyPath = flag.String("key_path", os.Getenv("KEY_PATH"), "Path to Key file (when using SSL for `prometheus_scrape_url`)")
skipServerCertCheck = flag.String("accept_invalid_cert", os.Getenv("ACCEPT_INVALID_CERT"), "Accept any certificate during TLS handshake. Insecure, use only for testing")
)
func main() {
flag.Parse()
if *cloudWatchNamespace == "" {
flag.PrintDefaults()
log.Fatal("prometheus-to-cloudwatch: Error: -cloudwatch_namespace or CLOUDWATCH_NAMESPACE required")
}
if *cloudWatchRegion == "" {
flag.PrintDefaults()
log.Fatal("prometheus-to-cloudwatch: Error: -cloudwatch_region or CLOUDWATCH_REGION required")
}
if *prometheusScrapeUrl == "" {
flag.PrintDefaults()
log.Fatal("prometheus-to-cloudwatch: Error: -prometheus_scrape_url or PROMETHEUS_SCRAPE_URL required")
}
if (*certPath != "" && *keyPath == "") || (*certPath == "" && *keyPath != "") {
flag.PrintDefaults()
log.Fatal("prometheus-to-cloudwatch: Error: when using SSL, both -prometheus_cert_path and -prometheus_key_path are required. If not using SSL, do not provide any of them")
}
var skipCertCheck = true
var err error
if *skipServerCertCheck != "" {
if skipCertCheck, err = strconv.ParseBool(*skipServerCertCheck); err != nil {
log.Fatal("prometheus-to-cloudwatch: Error: ", err)
}
}
config := &Config{
CloudWatchNamespace: *cloudWatchNamespace,
CloudWatchRegion: *cloudWatchRegion,
PrometheusScrapeUrl: *prometheusScrapeUrl,
PrometheusCertPath: *certPath,
PrometheusKeyPath: *keyPath,
PrometheusSkipServerCertCheck: skipCertCheck,
AwsAccessKeyId: *awsAccessKeyId,
AwsSecretAccessKey: *awsSecretAccessKey,
}
if *prometheusScrapeInterval != "" {
interval, err := strconv.Atoi(*prometheusScrapeInterval)
if err != nil {
log.Fatal("prometheus-to-cloudwatch: error parsing 'prometheus_scrape_interval': ", err)
}
config.CloudWatchPublishInterval = time.Duration(interval) * time.Second
}
if *cloudWatchPublishTimeout != "" {
timeout, err := strconv.Atoi(*cloudWatchPublishTimeout)
if err != nil {
log.Fatal("prometheus-to-cloudwatch: error parsing 'cloudwatch_publish_timeout': ", err)
}
config.CloudWatchPublishTimeout = time.Duration(timeout) * time.Second
}
bridge, err := NewBridge(config)
if err != nil {
log.Fatal("prometheus-to-cloudwatch: Error: ", err)
}
fmt.Println("prometheus-to-cloudwatch: Starting prometheus-to-cloudwatch bridge")
bridge.Run(context.Background())
}

350
prometheus_to_cloudwatch.go Normal file
View File

@ -0,0 +1,350 @@
package main
import (
"context"
"crypto/tls"
"errors"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/matttproud/golang_protobuf_extensions/pbutil"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
"github.com/prometheus/common/model"
"io"
"log"
"mime"
"net/http"
"sort"
"time"
)
const (
batchSize = 10
cwHighResLabel = "__cw_high_res"
cwUnitLabel = "__cw_unit"
acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3`
)
// Config defines configuration options
type Config struct {
// AWS access key Id with permissions to publish CloudWatch metrics
AwsAccessKeyId string
// AWS secret access key with permissions to publish CloudWatch metrics
AwsSecretAccessKey string
// Required. The CloudWatch namespace under which metrics should be published
CloudWatchNamespace string
// Required. The AWS Region to use
CloudWatchRegion string
// The frequency with which metrics should be published to Cloudwatch. Default: 15s
CloudWatchPublishInterval time.Duration
// Timeout for sending metrics to Cloudwatch. Default: 3s
CloudWatchPublishTimeout time.Duration
// Prometheus scrape URL
PrometheusScrapeUrl string
// Path to Certificate file
PrometheusCertPath string
// Path to Key file
PrometheusKeyPath string
// Accept any certificate during TLS handshake. Insecure, use only for testing
PrometheusSkipServerCertCheck bool
}
// Bridge pushes metrics to AWS CloudWatch
type Bridge struct {
cloudWatchPublishInterval time.Duration
cloudWatchNamespace string
cw *cloudwatch.CloudWatch
prometheusScrapeUrl string
prometheusCertPath string
prometheusKeyPath string
prometheusSkipServerCertCheck bool
}
// NewBridge initializes and returns a pointer to a Bridge using the
// supplied configuration, or an error if there is a problem with the configuration
func NewBridge(c *Config) (*Bridge, error) {
b := &Bridge{}
if c.CloudWatchNamespace == "" {
return nil, errors.New("CloudWatchNamespace required")
}
b.cloudWatchNamespace = c.CloudWatchNamespace
if c.PrometheusScrapeUrl == "" {
return nil, errors.New("PrometheusScrapeUrl required")
}
b.prometheusScrapeUrl = c.PrometheusScrapeUrl
b.prometheusCertPath = c.PrometheusCertPath
b.prometheusKeyPath = c.PrometheusKeyPath
b.prometheusSkipServerCertCheck = c.PrometheusSkipServerCertCheck
if c.CloudWatchPublishInterval > 0 {
b.cloudWatchPublishInterval = c.CloudWatchPublishInterval
} else {
b.cloudWatchPublishInterval = 30 * time.Second
}
var client = http.DefaultClient
if c.CloudWatchPublishTimeout > 0 {
client.Timeout = c.CloudWatchPublishTimeout
} else {
client.Timeout = 5 * time.Second
}
if c.CloudWatchRegion == "" {
return nil, errors.New("CloudWatchRegion required")
}
config := aws.NewConfig().WithHTTPClient(client).WithRegion(c.CloudWatchRegion)
// https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html
// https://docs.aws.amazon.com/sdk-for-go/api/aws/#Config
// If credentials are not provided in the variables, the chain of credential providers will search for credentials
// in environment variables, the shared credential file, and EC2 Instance Roles
if c.AwsAccessKeyId != "" && c.AwsSecretAccessKey != "" {
config.Credentials = credentials.NewStaticCredentials(c.AwsAccessKeyId, c.AwsSecretAccessKey, "")
}
sess, err := session.NewSession(config)
if err != nil {
return nil, err
}
b.cw = cloudwatch.New(sess)
return b, nil
}
// Run starts a loop that will push metrics to Cloudwatch at the configured interval. Accepts a context.Context to support cancellation
func (b *Bridge) Run(ctx context.Context) {
ticker := time.NewTicker(b.cloudWatchPublishInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
mfChan := make(chan *dto.MetricFamily, 1024)
go fetchMetricFamilies(b.prometheusScrapeUrl, mfChan, b.prometheusCertPath, b.prometheusKeyPath, b.prometheusSkipServerCertCheck)
var metricFamilies []*dto.MetricFamily
for mf := range mfChan {
metricFamilies = append(metricFamilies, mf)
}
err := b.publishMetricsToCloudWatch(metricFamilies)
if err != nil {
log.Println("prometheus-to-cloudwatch: error publishing to CloudWatch:", err)
} else {
log.Println(fmt.Sprintf("prometheus-to-cloudwatch: published %d metrics to CloudWatch", len(metricFamilies)))
}
case <-ctx.Done():
log.Println("prometheus-to-cloudwatch: stopping")
return
}
}
}
// NOTE: The CloudWatch API has the following limitations:
// - Max 40kb request size
// - Single namespace per request
// - Max 10 dimensions per metric
func (b *Bridge) publishMetricsToCloudWatch(mfs []*dto.MetricFamily) error {
vec, err := expfmt.ExtractSamples(&expfmt.DecodeOptions{Timestamp: model.Now()}, mfs...)
if err != nil {
return err
}
data := make([]*cloudwatch.MetricDatum, 0, batchSize)
for _, s := range vec {
name := getName(s.Metric)
data = appendDatum(data, name, s)
if len(data) == batchSize {
if err := b.flush(data); err != nil {
log.Println("prometheus-to-cloudwatch: error publishing to CloudWatch:", err)
}
data = make([]*cloudwatch.MetricDatum, 0, batchSize)
}
}
return b.flush(data)
}
func (b *Bridge) flush(data []*cloudwatch.MetricDatum) error {
if len(data) > 0 {
in := &cloudwatch.PutMetricDataInput{
MetricData: data,
Namespace: &b.cloudWatchNamespace,
}
_, err := b.cw.PutMetricData(in)
return err
}
return nil
}
func appendDatum(data []*cloudwatch.MetricDatum, name string, s *model.Sample) []*cloudwatch.MetricDatum {
metric := s.Metric
if len(metric) == 0 {
return data
}
d := &cloudwatch.MetricDatum{}
d.SetMetricName(name).
SetValue(float64(s.Value)).
SetTimestamp(s.Timestamp.Time()).
SetDimensions(getDimensions(metric)).
SetStorageResolution(getResolution(metric)).
SetUnit(getUnit(metric))
return append(data, d)
}
func getName(m model.Metric) string {
if n, ok := m[model.MetricNameLabel]; ok {
return string(n)
}
return ""
}
// getDimensions returns up to 10 dimensions for the provided metric - one for each label (except the __name__ label)
// If a metric has more than 10 labels, it attempts to behave deterministically and returning the first 10 labels as dimensions
func getDimensions(m model.Metric) []*cloudwatch.Dimension {
if len(m) == 0 {
return make([]*cloudwatch.Dimension, 0)
} else if _, ok := m[model.MetricNameLabel]; len(m) == 1 && ok {
return make([]*cloudwatch.Dimension, 0)
}
names := make([]string, 0, len(m))
for k := range m {
if !(k == model.MetricNameLabel || k == cwHighResLabel || k == cwUnitLabel) {
names = append(names, string(k))
}
}
sort.Strings(names)
dims := make([]*cloudwatch.Dimension, 0, len(names))
for _, name := range names {
if name != "" {
val := string(m[model.LabelName(name)])
if val != "" {
dims = append(dims, new(cloudwatch.Dimension).SetName(name).SetValue(val))
}
}
}
if len(dims) > 10 {
dims = dims[:10]
}
return dims
}
// Returns 1 if the metric contains a __cw_high_res label, otherwise returns 60
func getResolution(m model.Metric) int64 {
if _, ok := m[cwHighResLabel]; ok {
return 1
}
return 60
}
func getUnit(m model.Metric) string {
if u, ok := m[cwUnitLabel]; ok {
return string(u)
}
return "None"
}
// fetchMetricFamilies retrieves metrics from the provided URL, decodes them into MetricFamily proto messages, and sends them to the provided channel.
// It returns after all MetricFamilies have been sent
func fetchMetricFamilies(
url string, ch chan<- *dto.MetricFamily,
certificate string, key string,
skipServerCertCheck bool,
) {
defer close(ch)
var transport *http.Transport
if certificate != "" && key != "" {
cert, err := tls.LoadX509KeyPair(certificate, key)
if err != nil {
log.Fatal("prometheus-to-cloudwatch: Error: ", err)
}
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: skipServerCertCheck,
}
tlsConfig.BuildNameToCertificate()
transport = &http.Transport{TLSClientConfig: tlsConfig}
} else {
transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipServerCertCheck},
}
}
client := &http.Client{Transport: transport}
decodeContent(client, url, ch)
}
func decodeContent(client *http.Client, url string, ch chan<- *dto.MetricFamily) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalf("prometheus-to-cloudwatch: Error: creating GET request for URL %q failed: %s", url, err)
}
req.Header.Add("Accept", acceptHeader)
resp, err := client.Do(req)
if err != nil {
log.Fatalf("prometheus-to-cloudwatch: Error: executing GET request for URL %q failed: %s", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Fatalf("prometheus-to-cloudwatch: Error: GET request for URL %q returned HTTP status %s", url, resp.Status)
}
parseResponse(resp, ch)
}
// parseResponse consumes an http.Response and pushes it to the channel.
// It returns when all all MetricFamilies are parsed and put on the channel.
func parseResponse(resp *http.Response, ch chan<- *dto.MetricFamily) {
mediaType, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
if err == nil && mediaType == "application/vnd.google.protobuf" && params["encoding"] == "delimited" && params["proto"] == "io.prometheus.client.MetricFamily" {
for {
mf := &dto.MetricFamily{}
if _, err = pbutil.ReadDelimited(resp.Body, mf); err != nil {
if err == io.EOF {
break
}
log.Fatalln("prometheus-to-cloudwatch: Error: reading metric family protocol buffer failed:", err)
}
ch <- mf
}
} else {
var parser expfmt.TextParser
metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
if err != nil {
log.Fatalln("reading text format failed:", err)
}
for _, mf := range metricFamilies {
ch <- mf
}
}
}