feat: add ability to exclude dimensions per-metric (#28)

This commit is contained in:
Austin Cawley-Edwards 2019-10-10 10:14:29 -04:00 committed by Andriy Knysh
parent a0298ddbc1
commit 1c91a72bd9
4 changed files with 148 additions and 66 deletions

View File

@ -52,22 +52,23 @@ __NOTE__: The module accepts parameters as command-line arguments or as ENV vari
Command-line arguments take precedence over ENV vars
| Command-line argument | ENV var | Description |
|:-----------------------------|:-----------------------------|:------------------------------------------------------------------------------|
| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics |
| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics |
| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace |
| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region |
| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds |
| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds |
| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from |
| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) |
| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) |
| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing |
| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE |
| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... |
| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) |
| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) |
| Command-line argument | ENV var | Description |
|--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics |
| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics |
| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace |
| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region |
| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds |
| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds |
| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from |
| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) |
| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) |
| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing |
| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE |
| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... |
| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) |
| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) |
| exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Dimensions to exclude for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') |
__NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`)
@ -106,6 +107,7 @@ export ACCEPT_INVALID_CERT=true
# Optionally, restrict the subset of metrics to be exported to CloudWatch
# export INCLUDE_METRICS='jvm_*'
# export EXCLUDE_METRICS='jvm_memory_*,jvm_buffer_*'
# export EXCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod;jvm_buffer=job,pod'
./dist/bin/prometheus-to-cloudwatch
```
@ -136,6 +138,7 @@ docker run -i --rm \
-e ACCEPT_INVALID_CERT=true \
-e INCLUDE_METRICS="" \
-e EXCLUDE_METRICS="" \
-e EXCLUDE_DIMENSIONS_FOR_METRICS="" \
prometheus-to-cloudwatch
```
@ -316,19 +319,21 @@ Check out [our other projects][github], [follow us on twitter][twitter], [apply
### Contributors
| [![Erik Osterman][osterman_avatar]][osterman_homepage]<br/>[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]<br/>[Andriy Knysh][aknysh_homepage] | [![Igor Rodionov][goruha_avatar]][goruha_homepage]<br/>[Igor Rodionov][goruha_homepage] | [![yufukui-m][yufukui-m_avatar]][yufukui-m_homepage]<br/>[yufukui-m][yufukui-m_homepage] | [![Satadru Biswas][sbiswas-suplari_avatar]][sbiswas-suplari_homepage]<br/>[Satadru Biswas][sbiswas-suplari_homepage] |
|---|---|---|---|---|
| [![Erik Osterman][osterman_avatar]][osterman_homepage]<br/>[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]<br/>[Andriy Knysh][aknysh_homepage] | [![Igor Rodionov][goruha_avatar]][goruha_homepage]<br/>[Igor Rodionov][goruha_homepage] | [![yufukui-m][yufukui-m_avatar]][yufukui-m_homepage]<br/>[yufukui-m][yufukui-m_homepage] | [![Satadru Biswas][sbiswas-suplari_avatar]][sbiswas-suplari_homepage]<br/>[Satadru Biswas][sbiswas-suplari_homepage] | [![Austin ce][austince_avatar]][austince_homepage]<br/>[Austin ce][austince_homepage] |
|---|---|---|---|---|---|
[osterman_homepage]: https://github.com/osterman
[osterman_avatar]: https://github.com/osterman.png?size=150
[osterman_avatar]: https://img.cloudposse.com/150x150/https://github.com/osterman.png
[aknysh_homepage]: https://github.com/aknysh
[aknysh_avatar]: https://github.com/aknysh.png?size=150
[aknysh_avatar]: https://img.cloudposse.com/150x150/https://github.com/aknysh.png
[goruha_homepage]: https://github.com/goruha
[goruha_avatar]: https://github.com/goruha.png?size=150
[goruha_avatar]: https://img.cloudposse.com/150x150/https://github.com/goruha.png
[yufukui-m_homepage]: https://github.com/yufukui-m
[yufukui-m_avatar]: https://github.com/yufukui-m.png?size=150
[yufukui-m_avatar]: https://img.cloudposse.com/150x150/https://github.com/yufukui-m.png
[sbiswas-suplari_homepage]: https://github.com/sbiswas-suplari
[sbiswas-suplari_avatar]: https://github.com/sbiswas-suplari.png?size=150
[sbiswas-suplari_avatar]: https://img.cloudposse.com/150x150/https://github.com/sbiswas-suplari.png
[austince_homepage]: https://github.com/austince
[austince_avatar]: https://img.cloudposse.com/150x150/https://github.com/austince.png

View File

@ -46,22 +46,23 @@ usage: |-
Command-line arguments take precedence over ENV vars
| Command-line argument | ENV var | Description |
|:-----------------------------|:-----------------------------|:------------------------------------------------------------------------------|
| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics |
| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics |
| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace |
| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region |
| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds |
| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds |
| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from |
| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) |
| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) |
| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing |
| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE |
| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... |
| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) |
| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) |
| Command-line argument | ENV var | Description |
|--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics |
| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics |
| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace |
| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region |
| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds |
| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds |
| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from |
| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) |
| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) |
| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing |
| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE |
| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... |
| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) |
| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) |
| exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Dimensions to exclude for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') |
__NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`)
@ -96,6 +97,7 @@ examples: |-
# Optionally, restrict the subset of metrics to be exported to CloudWatch
# export INCLUDE_METRICS='jvm_*'
# export EXCLUDE_METRICS='jvm_memory_*,jvm_buffer_*'
# export EXCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod;jvm_buffer=job,pod'
./dist/bin/prometheus-to-cloudwatch
```
@ -126,6 +128,7 @@ examples: |-
-e ACCEPT_INVALID_CERT=true \
-e INCLUDE_METRICS="" \
-e EXCLUDE_METRICS="" \
-e EXCLUDE_DIMENSIONS_FOR_METRICS="" \
prometheus-to-cloudwatch
```
@ -191,3 +194,5 @@ contributors:
github: "yufukui-m"
- name: "Satadru Biswas"
github: "sbiswas-suplari"
- name: "Austin ce"
github: "austince"

93
main.go
View File

@ -15,22 +15,44 @@ import (
)
var (
awsAccessKeyId = flag.String("aws_access_key_id", os.Getenv("AWS_ACCESS_KEY_ID"), "AWS access key Id with permissions to publish CloudWatch metrics")
awsSecretAccessKey = flag.String("aws_secret_access_key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS secret access key with permissions to publish CloudWatch metrics")
cloudWatchNamespace = flag.String("cloudwatch_namespace", os.Getenv("CLOUDWATCH_NAMESPACE"), "CloudWatch Namespace")
cloudWatchRegion = flag.String("cloudwatch_region", os.Getenv("CLOUDWATCH_REGION"), "CloudWatch Region")
cloudWatchPublishTimeout = flag.String("cloudwatch_publish_timeout", os.Getenv("CLOUDWATCH_PUBLISH_TIMEOUT"), "CloudWatch publish timeout in seconds")
prometheusScrapeInterval = flag.String("prometheus_scrape_interval", os.Getenv("PROMETHEUS_SCRAPE_INTERVAL"), "Prometheus scrape interval in seconds")
prometheusScrapeUrl = flag.String("prometheus_scrape_url", os.Getenv("PROMETHEUS_SCRAPE_URL"), "Prometheus scrape URL")
certPath = flag.String("cert_path", os.Getenv("CERT_PATH"), "Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`)")
keyPath = flag.String("key_path", os.Getenv("KEY_PATH"), "Path to Key file (when using SSL for `prometheus_scrape_url`)")
skipServerCertCheck = flag.String("accept_invalid_cert", os.Getenv("ACCEPT_INVALID_CERT"), "Accept any certificate during TLS handshake. Insecure, use only for testing")
additionalDimension = flag.String("additional_dimension", os.Getenv("ADDITIONAL_DIMENSION"), "Additional dimension specified by NAME=VALUE")
replaceDimensions = flag.String("replace_dimensions", os.Getenv("REPLACE_DIMENSIONS"), "replace dimensions specified by NAME=VALUE,...")
includeMetrics = flag.String("include_metrics", os.Getenv("INCLUDE_METRICS"), "Only publish the specified metrics (comma-separated list of glob patterns, e.g. 'up,http_*')")
excludeMetrics = flag.String("exclude_metrics", os.Getenv("EXCLUDE_METRICS"), "Never publish the specified metrics (comma-separated list of glob patterns, e.g. 'tomcat_*')")
awsAccessKeyId = flag.String("aws_access_key_id", os.Getenv("AWS_ACCESS_KEY_ID"), "AWS access key Id with permissions to publish CloudWatch metrics")
awsSecretAccessKey = flag.String("aws_secret_access_key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS secret access key with permissions to publish CloudWatch metrics")
cloudWatchNamespace = flag.String("cloudwatch_namespace", os.Getenv("CLOUDWATCH_NAMESPACE"), "CloudWatch Namespace")
cloudWatchRegion = flag.String("cloudwatch_region", os.Getenv("CLOUDWATCH_REGION"), "CloudWatch Region")
cloudWatchPublishTimeout = flag.String("cloudwatch_publish_timeout", os.Getenv("CLOUDWATCH_PUBLISH_TIMEOUT"), "CloudWatch publish timeout in seconds")
prometheusScrapeInterval = flag.String("prometheus_scrape_interval", os.Getenv("PROMETHEUS_SCRAPE_INTERVAL"), "Prometheus scrape interval in seconds")
prometheusScrapeUrl = flag.String("prometheus_scrape_url", os.Getenv("PROMETHEUS_SCRAPE_URL"), "Prometheus scrape URL")
certPath = flag.String("cert_path", os.Getenv("CERT_PATH"), "Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`)")
keyPath = flag.String("key_path", os.Getenv("KEY_PATH"), "Path to Key file (when using SSL for `prometheus_scrape_url`)")
skipServerCertCheck = flag.String("accept_invalid_cert", os.Getenv("ACCEPT_INVALID_CERT"), "Accept any certificate during TLS handshake. Insecure, use only for testing")
additionalDimension = flag.String("additional_dimension", os.Getenv("ADDITIONAL_DIMENSION"), "Additional dimension specified by NAME=VALUE")
replaceDimensions = flag.String("replace_dimensions", os.Getenv("REPLACE_DIMENSIONS"), "replace dimensions specified by NAME=VALUE,...")
includeMetrics = flag.String("include_metrics", os.Getenv("INCLUDE_METRICS"), "Only publish the specified metrics (comma-separated list of glob patterns, e.g. 'up,http_*')")
excludeMetrics = flag.String("exclude_metrics", os.Getenv("EXCLUDE_METRICS"), "Never publish the specified metrics (comma-separated list of glob patterns, e.g. 'tomcat_*')")
excludeDimensionsForMetrics = flag.String("exclude_dimensions_for_metrics", os.Getenv("EXCLUDE_DIMENSIONS_FOR_METRICS"), "Dimensions to exclude for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;')")
)
// kevValMustParse takes a string and exits with a message if it cannot parse as KEY=VALUE
func keyValMustParse(str, message string) (string, string) {
kv := strings.SplitN(str, "=", 2)
if len(kv) != 2 {
log.Fatalf("prometheus-to-cloudwatch: Error: %s", message)
}
return kv[0], kv[1]
}
// stringSliceToSet creates a "set" (a boolean map) from a slice of strings
func stringSliceToSet(slice []string) StringSet {
boolMap := make(StringSet, len(slice))
for i := range slice {
boolMap[slice[i]] = true
}
return boolMap
}
func main() {
flag.Parse()
@ -62,11 +84,8 @@ func main() {
var additionalDimensions = map[string]string{}
if *additionalDimension != "" {
kv := strings.SplitN(*additionalDimension, "=", 2)
if len(kv) != 2 {
log.Fatal("prometheus-to-cloudwatch: Error: -additionalDimension must be formatted as NAME=VALUE")
}
additionalDimensions[kv[0]] = kv[1]
key, val := keyValMustParse(*additionalDimension, "-additionalDimension must be formatted as NAME=VALUE")
additionalDimensions[key] = val
}
var replaceDims = map[string]string{}
@ -74,11 +93,8 @@ func main() {
kvs := strings.Split(*replaceDimensions, ",")
if len(kvs) > 0 {
for _, rd := range kvs {
kv := strings.SplitN(rd, "=", 2)
if len(kv) != 2 {
log.Fatal("prometheus-to-cloudwatch: Error: -replaceDimensions must be formatted as NAME=VALUE,...")
}
replaceDims[kv[0]] = kv[1]
key, val := keyValMustParse(rd, "-replaceDimensions must be formatted as NAME=VALUE,...")
replaceDims[key] = val
}
}
}
@ -105,6 +121,34 @@ func main() {
}
}
var excludeDimensionsForMetricsList []MatcherWithStringSet
if *excludeDimensionsForMetrics != "" {
// split metric1=dim1,dim2;metric2=dim1
// into [
// metric1=dim1,dim2
// metric*=dim1
// ]
// then into [{ Matcher: "metric1": Set: [dim1, dim2] } , { Matcher: "metric_*": Set: [dim1] }]
for _, sublist := range strings.Split(*excludeDimensionsForMetrics, ";") {
key, val := keyValMustParse(sublist, "-exclude_dimensions_for_metrics must be formatted as METRIC_NAME=DIM_LIST;...")
metricPattern, err := glob.Compile(key)
if err != nil {
log.Fatal(fmt.Errorf("prometheus-to-cloudwatch: Error: -exclude_dimensions_for_metrics contains invalid glob pattern in '%s': %s", key, err))
}
dims := strings.Split(val, ",")
if len(dims) == 0 {
log.Fatalf("prometheus-to-cloudwatch: Error: -exclude_dimensions_for_metrics was not given dimensions to exclude for metric '%s'", key)
}
g := MatcherWithStringSet{
Matcher: metricPattern,
Set: stringSliceToSet(dims),
}
excludeDimensionsForMetricsList = append(excludeDimensionsForMetricsList, g)
}
}
config := &Config{
CloudWatchNamespace: *cloudWatchNamespace,
CloudWatchRegion: *cloudWatchRegion,
@ -118,6 +162,7 @@ func main() {
ReplaceDimensions: replaceDims,
IncludeMetrics: includeMetricsList,
ExcludeMetrics: excludeMetricsList,
ExcludeDimensionsForMetrics: excludeDimensionsForMetricsList,
}
if *prometheusScrapeInterval != "" {

View File

@ -33,6 +33,14 @@ const (
acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3`
)
type StringSet map[string]bool
// MatcherWithStringSet defines a Glob matcher with a set of associated strings
type MatcherWithStringSet struct {
Matcher glob.Glob
Set StringSet
}
// Config defines configuration options
type Config struct {
// AWS access key Id with permissions to publish CloudWatch metrics
@ -76,6 +84,9 @@ type Config struct {
// Never publish the specified metrics (a list of glob patterns, e.g. ["tomcat_*"])
ExcludeMetrics []glob.Glob
// Exclude certain dimensions from the specified metrics
ExcludeDimensionsForMetrics []MatcherWithStringSet
}
// Bridge pushes metrics to AWS CloudWatch
@ -91,8 +102,10 @@ type Bridge struct {
replaceDimensions map[string]string
includeMetrics []glob.Glob
excludeMetrics []glob.Glob
excludeDimensionsForMetrics []MatcherWithStringSet
}
// NewBridge initializes and returns a pointer to a Bridge using the
// supplied configuration, or an error if there is a problem with the configuration
func NewBridge(c *Config) (*Bridge, error) {
@ -115,6 +128,7 @@ func NewBridge(c *Config) (*Bridge, error) {
b.replaceDimensions = c.ReplaceDimensions
b.includeMetrics = c.IncludeMetrics
b.excludeMetrics = c.ExcludeMetrics
b.excludeDimensionsForMetrics = c.ExcludeDimensionsForMetrics
if c.CloudWatchPublishInterval > 0 {
b.cloudWatchPublishInterval = c.CloudWatchPublishInterval
@ -261,6 +275,16 @@ func (b *Bridge) shouldIgnoreMetric(metricName string) bool {
return true
}
// getDimensionsToExcludeSetForMetric gets the dimensions blacklist for a metric, or nil if there isn't one
func (b *Bridge) getDimensionsToExcludeSetForMetric(metricName string) StringSet {
for _, matcherWithSet := range b.excludeDimensionsForMetrics {
if matcherWithSet.Matcher.Match(metricName) {
return matcherWithSet.Set
}
}
return nil
}
func anyPatternMatches(patterns []glob.Glob, s string) bool {
for _, pattern := range patterns {
if pattern.Match(s) {
@ -355,9 +379,12 @@ func getDimensions(m model.Metric, num int, b *Bridge) ([]*cloudwatch.Dimension,
}
names := make([]string, 0, len(m))
for k := range m {
if !(k == model.MetricNameLabel || k == cwHighResLabel || k == cwUnitLabel) {
names = append(names, string(k))
excludeSet := b.getDimensionsToExcludeSetForMetric(string(m[model.MetricNameLabel]))
for dimName := range m {
if !(dimName == model.MetricNameLabel || dimName == cwHighResLabel || dimName == cwUnitLabel || (excludeSet != nil && excludeSet[string(dimName)])) {
names = append(names, string(dimName))
}
}