From 0564e214eecf0682e616b288f0cd3d12446bdbfe Mon Sep 17 00:00:00 2001 From: sbiswas-suplari <42981279+sbiswas-suplari@users.noreply.github.com> Date: Tue, 26 Feb 2019 11:28:42 -0800 Subject: [PATCH] Adding ability to replace dimensions to enable aggregation across dimensions (#12) * Adding ability to replace dimensions. This is useful to aggregate metrics across dimensions in the kube_state metrics namespace. The aggregation across dimensions is neccessary for adding CloudWatch alarms to kube-state metrics for the variable number of pods/container/nodes that will exist in our EKS clusters * Don't add replacement if not configured * Addressing PR comments: Add new flag and contributor --- README.md | 103 +++++++++++++++++++++++++++--------- README.yaml | 3 ++ chart/values.yaml | 1 + main.go | 16 ++++++ prometheus_to_cloudwatch.go | 48 ++++++++++++++--- 5 files changed, 138 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 67973b5..73b1e7b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ +[![README Header][readme_header_img]][readme_header_link] -[![Cloud Posse](https://cloudposse.com/logo-300x69.svg)](https://cloudposse.com) +[![Cloud Posse][logo]](https://cpco.io/homepage) # prometheus-to-cloudwatch [![Build Status](https://travis-ci.org/cloudposse/prometheus-to-cloudwatch.svg?branch=master)](https://travis-ci.org/cloudposse/prometheus-to-cloudwatch) [![Latest Release](https://img.shields.io/github/release/cloudposse/prometheus-to-cloudwatch.svg)](https://github.com/cloudposse/prometheus-to-cloudwatch/releases/latest) [![Slack Community](https://slack.cloudposse.com/badge.svg)](https://slack.cloudposse.com) @@ -11,7 +12,15 @@ Utility for scraping Prometheus metrics from a Prometheus client endpoint and pu --- -This project is part of our comprehensive ["SweetOps"](https://docs.cloudposse.com) approach towards DevOps. +This project is part of our comprehensive ["SweetOps"](https://cpco.io/sweetops) approach towards DevOps. +[][share_email] +[][share_googleplus] +[][share_facebook] +[][share_reddit] +[][share_linkedin] +[][share_twitter] + + It's 100% Open Source and licensed under the [APACHE2](LICENSE). @@ -23,6 +32,9 @@ It's 100% Open Source and licensed under the [APACHE2](LICENSE). + + + ## Screenshots @@ -51,6 +63,7 @@ Command-line arguments take precedence over ENV vars | keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) | | accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing | | additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE | +| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... | __NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`) @@ -164,6 +177,13 @@ helm install . +## Share the Love + +Like this project? Please give it a ★ on [our GitHub](https://github.com/cloudposse/prometheus-to-cloudwatch)! (it helps us **a lot**) + +Are you using this project or any of our other projects? Consider [leaving a testimonial][testimonial]. =) + + ## Related Projects Check out these related projects. @@ -180,26 +200,34 @@ Check out these related projects. File a GitHub [issue](https://github.com/cloudposse/prometheus-to-cloudwatch/issues), send us an [email][email] or join our [Slack Community][slack]. +[![README Commercial Support][readme_commercial_support_img]][readme_commercial_support_link] + ## Commercial Support Work directly with our team of DevOps experts via email, slack, and video conferencing. We provide [*commercial support*][commercial_support] for all of our [Open Source][github] projects. As a *Dedicated Support* customer, you have access to our team of subject matter experts at a fraction of the cost of a full-time engineer. -[![E-Mail](https://img.shields.io/badge/email-hello@cloudposse.com-blue.svg)](mailto:hello@cloudposse.com) +[![E-Mail](https://img.shields.io/badge/email-hello@cloudposse.com-blue.svg)][email] - **Questions.** We'll use a Shared Slack channel between your team and ours. - **Troubleshooting.** We'll help you triage why things aren't working. - **Code Reviews.** We'll review your Pull Requests and provide constructive feedback. - **Bug Fixes.** We'll rapidly work to fix any bugs in our projects. -- **Build New Terraform Modules.** We'll develop original modules to provision infrastructure. +- **Build New Terraform Modules.** We'll [develop original modules][module_development] to provision infrastructure. - **Cloud Architecture.** We'll assist with your cloud strategy and design. - **Implementation.** We'll provide hands-on support to implement our reference architectures. -## Community Forum -Get access to our [Open Source Community Forum][slack] on Slack. It's **FREE** to join for everyone! Our "SweetOps" community is where you get to talk with others who share a similar vision for how to rollout and manage infrastructure. This is the best place to talk shop, ask questions, solicit feedback, and work together as a community to build *sweet* infrastructure. + +## Slack Community + +Join our [Open Source Community][slack] on Slack. It's **FREE** for everyone! Our "SweetOps" community is where you get to talk with others who share a similar vision for how to rollout and manage infrastructure. This is the best place to talk shop, ask questions, solicit feedback, and work together as a community to build totally *sweet* infrastructure. + +## Newsletter + +Signup for [our newsletter][newsletter] that covers everything on our technology radar. Receive updates on what we're up to on GitHub as well as awesome new projects we discover. ## Contributing @@ -209,7 +237,7 @@ Please use the [issue tracker](https://github.com/cloudposse/prometheus-to-cloud ### Developing -If you are interested in being a contributor and want to get involved in developing this project or [help out](https://github.com/orgs/cloudposse/projects/3) with our other projects, we would love to hear from you! Shoot us an [email](mailto:hello@cloudposse.com). +If you are interested in being a contributor and want to get involved in developing this project or [help out](https://cpco.io/help-out) with our other projects, we would love to hear from you! Shoot us an [email][email]. In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. @@ -224,7 +252,7 @@ In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. ## Copyright -Copyright © 2017-2018 [Cloud Posse, LLC](https://cloudposse.com) +Copyright © 2017-2019 [Cloud Posse, LLC](https://cpco.io/copyright) @@ -265,32 +293,22 @@ All other trademarks referenced herein are the property of their respective owne ## About -This project is maintained and funded by [Cloud Posse, LLC][website]. Like it? Please let us know at +This project is maintained and funded by [Cloud Posse, LLC][website]. Like it? Please let us know by [leaving a testimonial][testimonial]! -[![Cloud Posse](https://cloudposse.com/logo-300x69.svg)](https://cloudposse.com) +[![Cloud Posse][logo]][website] -We're a [DevOps Professional Services][hire] company based in Los Angeles, CA. We love [Open Source Software](https://github.com/cloudposse/)! +We're a [DevOps Professional Services][hire] company based in Los Angeles, CA. We ❤️ [Open Source Software][we_love_open_source]. -We offer paid support on all of our projects. +We offer [paid support][commercial_support] on all of our projects. -Check out [our other projects][github], [apply for a job][jobs], or [hire us][hire] to help with your cloud strategy and implementation. +Check out [our other projects][github], [follow us on twitter][twitter], [apply for a job][jobs], or [hire us][hire] to help with your cloud strategy and implementation. - [docs]: https://docs.cloudposse.com/ - [website]: https://cloudposse.com/ - [github]: https://github.com/cloudposse/ - [commercial_support]: https://github.com/orgs/cloudposse/projects - [jobs]: https://cloudposse.com/jobs/ - [hire]: https://cloudposse.com/contact/ - [slack]: https://slack.cloudposse.com/ - [linkedin]: https://www.linkedin.com/company/cloudposse - [twitter]: https://twitter.com/cloudposse/ - [email]: mailto:hello@cloudposse.com ### Contributors -| [![Erik Osterman][osterman_avatar]][osterman_homepage]
[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]
[Andriy Knysh][aknysh_homepage] | [![Igor Rodionov][goruha_avatar]][goruha_homepage]
[Igor Rodionov][goruha_homepage] | [![yufukui-m][yufukui-m_avatar]][yufukui-m_homepage]
[yufukui-m][yufukui-m_homepage] | -|---|---|---|---| +| [![Erik Osterman][osterman_avatar]][osterman_homepage]
[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]
[Andriy Knysh][aknysh_homepage] | [![Igor Rodionov][goruha_avatar]][goruha_homepage]
[Igor Rodionov][goruha_homepage] | [![yufukui-m][yufukui-m_avatar]][yufukui-m_homepage]
[yufukui-m][yufukui-m_homepage] | [![Satadru Biswas][sbiswas-suplari_avatar]][sbiswas-suplari_homepage]
[Satadru Biswas][sbiswas-suplari_homepage] | +|---|---|---|---|---| [osterman_homepage]: https://github.com/osterman [osterman_avatar]: https://github.com/osterman.png?size=150 @@ -300,5 +318,40 @@ Check out [our other projects][github], [apply for a job][jobs], or [hire us][hi [goruha_avatar]: https://github.com/goruha.png?size=150 [yufukui-m_homepage]: https://github.com/yufukui-m [yufukui-m_avatar]: https://github.com/yufukui-m.png?size=150 + [sbiswas-suplari_homepage]: https://github.com/sbiswas-suplari + [sbiswas-suplari_avatar]: https://github.com/sbiswas-suplari.png?size=150 + +[![README Footer][readme_footer_img]][readme_footer_link] +[![Beacon][beacon]][website] + + [logo]: https://cloudposse.com/logo-300x69.svg + [docs]: https://cpco.io/docs + [website]: https://cpco.io/homepage + [github]: https://cpco.io/github + [jobs]: https://cpco.io/jobs + [hire]: https://cpco.io/hire + [slack]: https://cpco.io/slack + [linkedin]: https://cpco.io/linkedin + [twitter]: https://cpco.io/twitter + [testimonial]: https://cpco.io/leave-testimonial + [newsletter]: https://cpco.io/newsletter + [email]: https://cpco.io/email + [commercial_support]: https://cpco.io/commercial-support + [we_love_open_source]: https://cpco.io/we-love-open-source + [module_development]: https://cpco.io/module-development + [terraform_modules]: https://cpco.io/terraform-modules + [readme_header_img]: https://cloudposse.com/readme/header/img?repo=cloudposse/prometheus-to-cloudwatch + [readme_header_link]: https://cloudposse.com/readme/header/link?repo=cloudposse/prometheus-to-cloudwatch + [readme_footer_img]: https://cloudposse.com/readme/footer/img?repo=cloudposse/prometheus-to-cloudwatch + [readme_footer_link]: https://cloudposse.com/readme/footer/link?repo=cloudposse/prometheus-to-cloudwatch + [readme_commercial_support_img]: https://cloudposse.com/readme/commercial-support/img?repo=cloudposse/prometheus-to-cloudwatch + [readme_commercial_support_link]: https://cloudposse.com/readme/commercial-support/link?repo=cloudposse/prometheus-to-cloudwatch + [share_twitter]: https://twitter.com/intent/tweet/?text=prometheus-to-cloudwatch&url=https://github.com/cloudposse/prometheus-to-cloudwatch + [share_linkedin]: https://www.linkedin.com/shareArticle?mini=true&title=prometheus-to-cloudwatch&url=https://github.com/cloudposse/prometheus-to-cloudwatch + [share_reddit]: https://reddit.com/submit/?url=https://github.com/cloudposse/prometheus-to-cloudwatch + [share_facebook]: https://facebook.com/sharer/sharer.php?u=https://github.com/cloudposse/prometheus-to-cloudwatch + [share_googleplus]: https://plus.google.com/share?url=https://github.com/cloudposse/prometheus-to-cloudwatch + [share_email]: mailto:?subject=prometheus-to-cloudwatch&body=https://github.com/cloudposse/prometheus-to-cloudwatch + [beacon]: https://ga-beacon.cloudposse.com/UA-76589703-4/cloudposse/prometheus-to-cloudwatch?pixel&cs=github&cm=readme&an=prometheus-to-cloudwatch diff --git a/README.yaml b/README.yaml index f964abc..346cf4f 100644 --- a/README.yaml +++ b/README.yaml @@ -59,6 +59,7 @@ usage: |- | keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) | | accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing | | additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE | + | replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... | __NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`) @@ -181,3 +182,5 @@ contributors: github: "goruha" - name: "yufukui-m" github: "yufukui-m" + - name: "Satadru Biswas" + github: "sbiswas-suplari" diff --git a/chart/values.yaml b/chart/values.yaml index d7c5dbf..33f6213 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -11,6 +11,7 @@ env: CERT_PATH: "" KEY_PATH: "" ACCEPT_INVALID_CERT: "true" + REPLACE_DIMENSIONS: "xxx=yyy,aaa=bbb" secrets: AWS_ACCESS_KEY_ID: diff --git a/main.go b/main.go index b8f45c2..e2e8e40 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ var ( keyPath = flag.String("key_path", os.Getenv("KEY_PATH"), "Path to Key file (when using SSL for `prometheus_scrape_url`)") skipServerCertCheck = flag.String("accept_invalid_cert", os.Getenv("ACCEPT_INVALID_CERT"), "Accept any certificate during TLS handshake. Insecure, use only for testing") additionalDimension = flag.String("additional_dimension", os.Getenv("ADDITIONAL_DIMENSION"), "Additional dimension specified by NAME=VALUE") + replaceDimensions = flag.String("replace_dimensions", os.Getenv("REPLACE_DIMENSIONS"), "replace dimensions specified by NAME=VALUE,...") ) func main() { @@ -63,6 +64,20 @@ func main() { additionalDimensions[kv[0]] = kv[1] } + var replaceDims = map[string]string{} + if *replaceDimensions != "" { + kvs := strings.Split(*replaceDimensions, ",") + if len(kvs) > 0 { + for _, rd := range kvs { + kv := strings.SplitN(rd, "=", 2) + if len(kv) != 2 { + log.Fatal("prometheus-to-cloudwatch: Error: -replaceDimensions must be formated as NAME=VALUE,...") + } + replaceDims[kv[0]] = kv[1] + } + } + } + config := &Config{ CloudWatchNamespace: *cloudWatchNamespace, CloudWatchRegion: *cloudWatchRegion, @@ -73,6 +88,7 @@ func main() { AwsAccessKeyId: *awsAccessKeyId, AwsSecretAccessKey: *awsSecretAccessKey, AdditionalDimensions: additionalDimensions, + ReplaceDimensions: replaceDims, } if *prometheusScrapeInterval != "" { diff --git a/prometheus_to_cloudwatch.go b/prometheus_to_cloudwatch.go index a91482a..000c86e 100644 --- a/prometheus_to_cloudwatch.go +++ b/prometheus_to_cloudwatch.go @@ -62,6 +62,9 @@ type Config struct { // Additional dimensions to send to CloudWatch AdditionalDimensions map[string]string + + // Replace dimensions with the provided label. This allows for aggregating metrics across dimensions so we can set CloudWatch Alarms on the metrics + ReplaceDimensions map[string]string } // Bridge pushes metrics to AWS CloudWatch @@ -74,6 +77,7 @@ type Bridge struct { prometheusKeyPath string prometheusSkipServerCertCheck bool additionalDimensions map[string]string + replaceDimensions map[string]string } // NewBridge initializes and returns a pointer to a Bridge using the @@ -95,6 +99,7 @@ func NewBridge(c *Config) (*Bridge, error) { b.prometheusKeyPath = c.PrometheusKeyPath b.prometheusSkipServerCertCheck = c.PrometheusSkipServerCertCheck b.additionalDimensions = c.AdditionalDimensions + b.replaceDimensions = c.ReplaceDimensions if c.CloudWatchPublishInterval > 0 { b.cloudWatchPublishInterval = c.CloudWatchPublishInterval @@ -211,16 +216,30 @@ func appendDatum(data []*cloudwatch.MetricDatum, name string, s *model.Sample, b return data } - d := &cloudwatch.MetricDatum{} + datum := &cloudwatch.MetricDatum{} - d.SetMetricName(name). + kubeStateDimensions, replacedDimensions := getDimensions(metric, 10-len(b.additionalDimensions), b) + datum.SetMetricName(name). SetValue(float64(s.Value)). SetTimestamp(s.Timestamp.Time()). - SetDimensions(append(getDimensions(metric, 10-len(b.additionalDimensions)), getAdditionalDimensions(b)...)). + SetDimensions(append(kubeStateDimensions, getAdditionalDimensions(b)...)). SetStorageResolution(getResolution(metric)). SetUnit(getUnit(metric)) + data = append(data, datum) - return append(data, d) + // Don't add replacement if not configured + if replacedDimensions != nil && len(replacedDimensions) > 0 { + replacedDimensionDatum := &cloudwatch.MetricDatum{} + replacedDimensionDatum.SetMetricName(name). + SetValue(float64(s.Value)). + SetTimestamp(s.Timestamp.Time()). + SetDimensions(append(replacedDimensions, getAdditionalDimensions(b)...)). + SetStorageResolution(getResolution(metric)). + SetUnit(getUnit(metric)) + data = append(data, replacedDimensionDatum) + } + + return data } func getName(m model.Metric) string { @@ -232,11 +251,11 @@ func getName(m model.Metric) string { // getDimensions returns up to 10 dimensions for the provided metric - one for each label (except the __name__ label) // If a metric has more than 10 labels, it attempts to behave deterministically and returning the first 10 labels as dimensions -func getDimensions(m model.Metric, num int) []*cloudwatch.Dimension { +func getDimensions(m model.Metric, num int, b *Bridge) ([]*cloudwatch.Dimension, []*cloudwatch.Dimension) { if len(m) == 0 { - return make([]*cloudwatch.Dimension, 0) + return make([]*cloudwatch.Dimension, 0), nil } else if _, ok := m[model.MetricNameLabel]; len(m) == 1 && ok { - return make([]*cloudwatch.Dimension, 0) + return make([]*cloudwatch.Dimension, 0), nil } names := make([]string, 0, len(m)) @@ -248,12 +267,21 @@ func getDimensions(m model.Metric, num int) []*cloudwatch.Dimension { sort.Strings(names) dims := make([]*cloudwatch.Dimension, 0, len(names)) + replacedDims := make([]*cloudwatch.Dimension, 0, len(names)) for _, name := range names { if name != "" { val := string(m[model.LabelName(name)]) if val != "" { dims = append(dims, new(cloudwatch.Dimension).SetName(name).SetValue(val)) + // Don't add replacement if not configured + if b.replaceDimensions != nil && len(b.replaceDimensions) > 0 { + if replacement, ok := b.replaceDimensions[name]; ok { + replacedDims = append(replacedDims, new(cloudwatch.Dimension).SetName(name).SetValue(replacement)) + } else { + replacedDims = append(replacedDims, new(cloudwatch.Dimension).SetName(name).SetValue(val)) + } + } } } } @@ -262,7 +290,11 @@ func getDimensions(m model.Metric, num int) []*cloudwatch.Dimension { dims = dims[:num] } - return dims + if len(replacedDims) > num { + replacedDims = replacedDims[:num] + } + + return dims, replacedDims } func getAdditionalDimensions(b *Bridge) []*cloudwatch.Dimension {