From 06ce0f393f6bb07becfebea201326461b1fed9dd Mon Sep 17 00:00:00 2001 From: Austin Cawley-Edwards Date: Mon, 14 Oct 2019 14:24:14 -0400 Subject: [PATCH] feat: add ability to include dimensions per-metric (#29) * feat: add ability to include dimensions per-metric * docs: regenerate readme * fix: invert inclusion list order * style: spacing * docs: make include/exclude dimension logic more clear --- README.md | 37 ++++++++++--------- README.yaml | 37 ++++++++++--------- chart/templates/NOTES.txt | 2 ++ main.go | 67 ++++++++++++++++++++-------------- prometheus_to_cloudwatch.go | 72 ++++++++++++++++++++++++++----------- 5 files changed, 135 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 4d0d0f2..eec7ad4 100644 --- a/README.md +++ b/README.md @@ -52,23 +52,24 @@ __NOTE__: The module accepts parameters as command-line arguments or as ENV vari Command-line arguments take precedence over ENV vars -| Command-line argument | ENV var | Description | -|--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics | -| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics | -| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace | -| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region | -| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds | -| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds | -| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from | -| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) | -| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) | -| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing | -| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE | -| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... | -| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) | -| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) | -| exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Dimensions to exclude for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') | +| Command-line argument | ENV var | Description | +|--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics | +| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics | +| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace | +| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region | +| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds | +| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds | +| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from | +| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) | +| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) | +| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing | +| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE | +| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... | +| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) | +| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) | +| include_dimensions_for_metrics | INCLUDE_DIMENSIONS_FOR_METRICS | Only publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job_id') | +| exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Never publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') | __NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`) @@ -107,6 +108,7 @@ export ACCEPT_INVALID_CERT=true # Optionally, restrict the subset of metrics to be exported to CloudWatch # export INCLUDE_METRICS='jvm_*' # export EXCLUDE_METRICS='jvm_memory_*,jvm_buffer_*' +# export INCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod_id' # export EXCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod;jvm_buffer=job,pod' ./dist/bin/prometheus-to-cloudwatch @@ -138,6 +140,7 @@ docker run -i --rm \ -e ACCEPT_INVALID_CERT=true \ -e INCLUDE_METRICS="" \ -e EXCLUDE_METRICS="" \ + -e INCLUDE_DIMENSIONS_FOR_METRICS="" \ -e EXCLUDE_DIMENSIONS_FOR_METRICS="" \ prometheus-to-cloudwatch ``` diff --git a/README.yaml b/README.yaml index 64292b7..faf3c56 100644 --- a/README.yaml +++ b/README.yaml @@ -46,23 +46,24 @@ usage: |- Command-line arguments take precedence over ENV vars - | Command-line argument | ENV var | Description | - |--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| - | aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics | - | aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics | - | cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace | - | cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region | - | cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds | - | prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds | - | prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from | - | cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) | - | keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) | - | accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing | - | additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE | - | replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... | - | include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) | - | exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) | - | exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Dimensions to exclude for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') | + | Command-line argument | ENV var | Description | + |--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics | + | aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics | + | cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace | + | cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region | + | cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds | + | prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds | + | prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from | + | cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) | + | keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) | + | accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing | + | additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE | + | replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... | + | include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) | + | exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) | + | include_dimensions_for_metrics | INCLUDE_DIMENSIONS_FOR_METRICS | Only publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job_id') | + | exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Never publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') | __NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`) @@ -97,6 +98,7 @@ examples: |- # Optionally, restrict the subset of metrics to be exported to CloudWatch # export INCLUDE_METRICS='jvm_*' # export EXCLUDE_METRICS='jvm_memory_*,jvm_buffer_*' + # export INCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod_id' # export EXCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod;jvm_buffer=job,pod' ./dist/bin/prometheus-to-cloudwatch @@ -128,6 +130,7 @@ examples: |- -e ACCEPT_INVALID_CERT=true \ -e INCLUDE_METRICS="" \ -e EXCLUDE_METRICS="" \ + -e INCLUDE_DIMENSIONS_FOR_METRICS="" \ -e EXCLUDE_DIMENSIONS_FOR_METRICS="" \ prometheus-to-cloudwatch ``` diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index cc8016d..7166be6 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -12,6 +12,8 @@ env: # Optionally: - INCLUDE_METRICS - EXCLUDE_METRICS + - INCLUDE_DIMENSIONS_FOR_METRICS + - EXCLUDE_DIMENSIONS_FOR_METRICS secrets: - AWS_ACCESS_KEY_ID diff --git a/main.go b/main.go index 27500a2..e10659d 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "context" "flag" "fmt" - "github.com/gobwas/glob" "log" "os" "os/signal" @@ -12,6 +11,8 @@ import ( "strings" "syscall" "time" + + "github.com/gobwas/glob" ) var ( @@ -29,7 +30,8 @@ var ( replaceDimensions = flag.String("replace_dimensions", os.Getenv("REPLACE_DIMENSIONS"), "replace dimensions specified by NAME=VALUE,...") includeMetrics = flag.String("include_metrics", os.Getenv("INCLUDE_METRICS"), "Only publish the specified metrics (comma-separated list of glob patterns, e.g. 'up,http_*')") excludeMetrics = flag.String("exclude_metrics", os.Getenv("EXCLUDE_METRICS"), "Never publish the specified metrics (comma-separated list of glob patterns, e.g. 'tomcat_*')") - excludeDimensionsForMetrics = flag.String("exclude_dimensions_for_metrics", os.Getenv("EXCLUDE_DIMENSIONS_FOR_METRICS"), "Dimensions to exclude for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;')") + includeDimensionsForMetrics = flag.String("include_dimensions_for_metrics", os.Getenv("INCLUDE_DIMENSIONS_FOR_METRICS"), "Only publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job_id')") + excludeDimensionsForMetrics = flag.String("exclude_dimensions_for_metrics", os.Getenv("EXCLUDE_DIMENSIONS_FOR_METRICS"), "Never publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;')") ) // kevValMustParse takes a string and exits with a message if it cannot parse as KEY=VALUE @@ -41,6 +43,37 @@ func keyValMustParse(str, message string) (string, string) { return kv[0], kv[1] } +// dimensionMatcherListMustParse takes a string and a flag name and exists with a message +// if it cannot parse as GLOB=dim1,dim2;GLOB2=dim3 +func dimensionMatcherListMustParse(str, flag string) []MatcherWithStringSet { + var matcherList []MatcherWithStringSet + // split metric1=dim1,dim2;metric2=dim1 + // into [ + // metric1=dim1,dim2 + // metric*=dim1 + // ] + // then into [{ Matcher: "metric1": Set: [dim1, dim2] } , { Matcher: "metric_*": Set: [dim1] }] + for _, sublist := range strings.Split(str, ";") { + key, val := keyValMustParse(sublist, fmt.Sprintf("%s must be formatted as METRIC_NAME=DIM_LIST;...", flag)) + + metricPattern, err := glob.Compile(key) + if err != nil { + log.Fatal(fmt.Errorf("prometheus-to-cloudwatch: Error: %s contains invalid glob pattern in '%s': %s", flag, key, err)) + } + + dims := strings.Split(val, ",") + if len(dims) == 0 { + log.Fatalf("prometheus-to-cloudwatch: Error: %s was not given dimensions to exclude for metric '%s'", flag, key) + } + g := MatcherWithStringSet{ + Matcher: metricPattern, + Set: stringSliceToSet(dims), + } + matcherList = append(matcherList, g) + } + return matcherList +} + // stringSliceToSet creates a "set" (a boolean map) from a slice of strings func stringSliceToSet(slice []string) StringSet { boolMap := make(StringSet, len(slice)) @@ -52,7 +85,6 @@ func stringSliceToSet(slice []string) StringSet { return boolMap } - func main() { flag.Parse() @@ -123,30 +155,12 @@ func main() { var excludeDimensionsForMetricsList []MatcherWithStringSet if *excludeDimensionsForMetrics != "" { - // split metric1=dim1,dim2;metric2=dim1 - // into [ - // metric1=dim1,dim2 - // metric*=dim1 - // ] - // then into [{ Matcher: "metric1": Set: [dim1, dim2] } , { Matcher: "metric_*": Set: [dim1] }] - for _, sublist := range strings.Split(*excludeDimensionsForMetrics, ";") { - key, val := keyValMustParse(sublist, "-exclude_dimensions_for_metrics must be formatted as METRIC_NAME=DIM_LIST;...") + excludeDimensionsForMetricsList = dimensionMatcherListMustParse(*excludeDimensionsForMetrics, "-exclude_dimensions_for_metrics") + } - metricPattern, err := glob.Compile(key) - if err != nil { - log.Fatal(fmt.Errorf("prometheus-to-cloudwatch: Error: -exclude_dimensions_for_metrics contains invalid glob pattern in '%s': %s", key, err)) - } - - dims := strings.Split(val, ",") - if len(dims) == 0 { - log.Fatalf("prometheus-to-cloudwatch: Error: -exclude_dimensions_for_metrics was not given dimensions to exclude for metric '%s'", key) - } - g := MatcherWithStringSet{ - Matcher: metricPattern, - Set: stringSliceToSet(dims), - } - excludeDimensionsForMetricsList = append(excludeDimensionsForMetricsList, g) - } + var includeDimensionsForMetricsList []MatcherWithStringSet + if *includeDimensionsForMetrics != "" { + includeDimensionsForMetricsList = dimensionMatcherListMustParse(*includeDimensionsForMetrics, "-include_dimensions_for_metrics") } config := &Config{ @@ -163,6 +177,7 @@ func main() { IncludeMetrics: includeMetricsList, ExcludeMetrics: excludeMetricsList, ExcludeDimensionsForMetrics: excludeDimensionsForMetricsList, + IncludeDimensionsForMetrics: includeDimensionsForMetricsList, } if *prometheusScrapeInterval != "" { diff --git a/prometheus_to_cloudwatch.go b/prometheus_to_cloudwatch.go index ab46f90..dbd7580 100644 --- a/prometheus_to_cloudwatch.go +++ b/prometheus_to_cloudwatch.go @@ -7,6 +7,14 @@ import ( "crypto/tls" "errors" "fmt" + "io" + "log" + "math" + "mime" + "net/http" + "sort" + "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/request" @@ -17,13 +25,6 @@ import ( dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" - "io" - "log" - "math" - "mime" - "net/http" - "sort" - "time" ) const ( @@ -41,6 +42,20 @@ type MatcherWithStringSet struct { Set StringSet } +// getMatchingSet returns the first set that matches a string, or nil if there is no match +func getMatchingSet(matcherSets []MatcherWithStringSet, str string) StringSet { + if matcherSets == nil { + return nil + } + + for _, matcherWithSet := range matcherSets { + if matcherWithSet.Matcher.Match(str) { + return matcherWithSet.Set + } + } + return nil +} + // Config defines configuration options type Config struct { // AWS access key Id with permissions to publish CloudWatch metrics @@ -85,6 +100,9 @@ type Config struct { // Never publish the specified metrics (a list of glob patterns, e.g. ["tomcat_*"]) ExcludeMetrics []glob.Glob + // Only publish certain dimensions from the specified metrics + IncludeDimensionsForMetrics []MatcherWithStringSet + // Exclude certain dimensions from the specified metrics ExcludeDimensionsForMetrics []MatcherWithStringSet } @@ -102,10 +120,10 @@ type Bridge struct { replaceDimensions map[string]string includeMetrics []glob.Glob excludeMetrics []glob.Glob + includeDimensionsForMetrics []MatcherWithStringSet excludeDimensionsForMetrics []MatcherWithStringSet } - // NewBridge initializes and returns a pointer to a Bridge using the // supplied configuration, or an error if there is a problem with the configuration func NewBridge(c *Config) (*Bridge, error) { @@ -128,6 +146,7 @@ func NewBridge(c *Config) (*Bridge, error) { b.replaceDimensions = c.ReplaceDimensions b.includeMetrics = c.IncludeMetrics b.excludeMetrics = c.ExcludeMetrics + b.includeDimensionsForMetrics = c.IncludeDimensionsForMetrics b.excludeDimensionsForMetrics = c.ExcludeDimensionsForMetrics if c.CloudWatchPublishInterval > 0 { @@ -275,16 +294,6 @@ func (b *Bridge) shouldIgnoreMetric(metricName string) bool { return true } -// getDimensionsToExcludeSetForMetric gets the dimensions blacklist for a metric, or nil if there isn't one -func (b *Bridge) getDimensionsToExcludeSetForMetric(metricName string) StringSet { - for _, matcherWithSet := range b.excludeDimensionsForMetrics { - if matcherWithSet.Matcher.Match(metricName) { - return matcherWithSet.Set - } - } - return nil -} - func anyPatternMatches(patterns []glob.Glob, s string) bool { for _, pattern := range patterns { if pattern.Match(s) { @@ -369,6 +378,27 @@ func getName(m model.Metric) string { return "" } +// shouldIncludeDimension determines whether or not to keep this dimension when publishing to cloudwatch +// if an `includeSet` is specified, this will only return `true` for dimensions in that set +func shouldIncludeDimension(dimName model.LabelName, includeSet, excludeSet StringSet) bool { + if dimName == model.MetricNameLabel || dimName == cwHighResLabel || dimName == cwUnitLabel { + return false + } + + dimNameStr := string(dimName) + + // blacklist first + if excludeSet != nil && excludeSet[dimNameStr] { + return false + } + // if no whitelist, keep it as it passed the blacklist + if includeSet == nil { + return true + } + // otherwise, check the whitelist + return includeSet[dimNameStr] +} + // getDimensions returns up to 10 dimensions for the provided metric - one for each label (except the __name__ label) // If a metric has more than 10 labels, it attempts to behave deterministically and returning the first 10 labels as dimensions func getDimensions(m model.Metric, num int, b *Bridge) ([]*cloudwatch.Dimension, []*cloudwatch.Dimension) { @@ -380,10 +410,12 @@ func getDimensions(m model.Metric, num int, b *Bridge) ([]*cloudwatch.Dimension, names := make([]string, 0, len(m)) - excludeSet := b.getDimensionsToExcludeSetForMetric(string(m[model.MetricNameLabel])) + metricName := getName(m) + includeSet := getMatchingSet(b.includeDimensionsForMetrics, metricName) + excludeSet := getMatchingSet(b.excludeDimensionsForMetrics, metricName) for dimName := range m { - if !(dimName == model.MetricNameLabel || dimName == cwHighResLabel || dimName == cwUnitLabel || (excludeSet != nil && excludeSet[string(dimName)])) { + if shouldIncludeDimension(dimName, includeSet, excludeSet) { names = append(names, string(dimName)) } }