ai-workspace-infra/artifacts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add ability to include dimensions per-metric (#29)

Browse Source 
* feat: add ability to include dimensions per-metric

* docs: regenerate readme

* fix: invert inclusion list order

* style: spacing

* docs: make include/exclude dimension logic more clear
This commit is contained in:
Austin Cawley-Edwards 2019-10-14 14:24:14 -04:00 committed by Andriy Knysh
parent 1c91a72bd9
commit 06ce0f393f
5 changed files with 135 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
README.md
View File

@ -52,23 +52,24 @@ __NOTE__: The module accepts parameters as command-line arguments or as ENV vari
Command-line arguments take precedence over ENV vars
| Command-line argument | ENV var | Description |
|--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics |
| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics |
| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace |
| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region |
| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds |
| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds |
| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from |
| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) |
| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) |
| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing |
| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE |
| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... |
| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) |
| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) |
| exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Dimensions to exclude for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') |
| Command-line argument | ENV var | Description |
|--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics |
| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics |
| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace |
| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region |
| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds |
| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds |
| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from |
| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) |
| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) |
| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing |
| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE |
| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... |
| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) |
| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) |
| include_dimensions_for_metrics | INCLUDE_DIMENSIONS_FOR_METRICS | Only publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job_id') |
| exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Never publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') |
__NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`)
@ -107,6 +108,7 @@ export ACCEPT_INVALID_CERT=true
# Optionally, restrict the subset of metrics to be exported to CloudWatch
# export INCLUDE_METRICS='jvm_*'
# export EXCLUDE_METRICS='jvm_memory_*,jvm_buffer_*'
# export INCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod_id'
# export EXCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod;jvm_buffer=job,pod'
./dist/bin/prometheus-to-cloudwatch
@ -138,6 +140,7 @@ docker run -i --rm \
-e ACCEPT_INVALID_CERT=true \
-e INCLUDE_METRICS="" \
-e EXCLUDE_METRICS="" \
-e INCLUDE_DIMENSIONS_FOR_METRICS="" \
-e EXCLUDE_DIMENSIONS_FOR_METRICS="" \
prometheus-to-cloudwatch
```

37
README.yaml
View File

@ -46,23 +46,24 @@ usage: |-
Command-line arguments take precedence over ENV vars
| Command-line argument | ENV var | Description |
|--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics |
| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics |
| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace |
| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region |
| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds |
| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds |
| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from |
| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) |
| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) |
| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing |
| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE |
| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... |
| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) |
| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) |
| exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Dimensions to exclude for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') |
| Command-line argument | ENV var | Description |
|--------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| aws_access_key_id | AWS_ACCESS_KEY_ID | AWS access key Id with permissions to publish CloudWatch metrics |
| aws_secret_access_key | AWS_SECRET_ACCESS_KEY | AWS secret access key with permissions to publish CloudWatch metrics |
| cloudwatch_namespace | CLOUDWATCH_NAMESPACE | CloudWatch Namespace |
| cloudwatch_region | CLOUDWATCH_REGION | CloudWatch AWS Region |
| cloudwatch_publish_timeout | CLOUDWATCH_PUBLISH_TIMEOUT | CloudWatch publish timeout in seconds |
| prometheus_scrape_interval | PROMETHEUS_SCRAPE_INTERVAL | Prometheus scrape interval in seconds |
| prometheus_scrape_url | PROMETHEUS_SCRAPE_URL | The URL to scrape Prometheus metrics from |
| cert_path | CERT_PATH | Path to SSL Certificate file (when using SSL for `prometheus_scrape_url`) |
| keyPath | KEY_PATH | Path to Key file (when using SSL for `prometheus_scrape_url`) |
| accept_invalid_cert | ACCEPT_INVALID_CERT | Accept any certificate during TLS handshake. Insecure, use only for testing |
| additional_dimension | ADDITIONAL_DIMENSION | Additional dimension specified by NAME=VALUE |
| replace_dimensions | REPLACE_DIMENSIONS | Replace dimensions specified by NAME=VALUE,... |
| include_metrics | INCLUDE_METRICS | Only publish the specified metrics (comma-separated list of glob patterns) |
| exclude_metrics | EXCLUDE_METRICS | Never publish the specified metrics (comma-separated list of glob patterns) |
| include_dimensions_for_metrics | INCLUDE_DIMENSIONS_FOR_METRICS | Only publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job_id') |
| exclude_dimensions_for_metrics | EXCLUDE_DIMENSIONS_FOR_METRICS | Never publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;') |
__NOTE__: If AWS credentials are not provided in the command-line arguments (`aws_access_key_id` and `aws_secret_access_key`)
@ -97,6 +98,7 @@ examples: |-
# Optionally, restrict the subset of metrics to be exported to CloudWatch
# export INCLUDE_METRICS='jvm_*'
# export EXCLUDE_METRICS='jvm_memory_*,jvm_buffer_*'
# export INCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod_id'
# export EXCLUDE_DIMENSIONS_FOR_METRICS='jvm_memory_*=pod;jvm_buffer=job,pod'
./dist/bin/prometheus-to-cloudwatch
@ -128,6 +130,7 @@ examples: |-
-e ACCEPT_INVALID_CERT=true \
-e INCLUDE_METRICS="" \
-e EXCLUDE_METRICS="" \
-e INCLUDE_DIMENSIONS_FOR_METRICS="" \
-e EXCLUDE_DIMENSIONS_FOR_METRICS="" \
prometheus-to-cloudwatch
```

2
chart/templates/NOTES.txt
View File

@ -12,6 +12,8 @@ env:
# Optionally:
- INCLUDE_METRICS
- EXCLUDE_METRICS
- INCLUDE_DIMENSIONS_FOR_METRICS
- EXCLUDE_DIMENSIONS_FOR_METRICS
secrets:
- AWS_ACCESS_KEY_ID

67
main.go
View File

@ -4,7 +4,6 @@ import (
"context"
"flag"
"fmt"
"github.com/gobwas/glob"
"log"
"os"
"os/signal"
@ -12,6 +11,8 @@ import (
"strings"
"syscall"
"time"
"github.com/gobwas/glob"
)
var (
@ -29,7 +30,8 @@ var (
replaceDimensions = flag.String("replace_dimensions", os.Getenv("REPLACE_DIMENSIONS"), "replace dimensions specified by NAME=VALUE,...")
includeMetrics = flag.String("include_metrics", os.Getenv("INCLUDE_METRICS"), "Only publish the specified metrics (comma-separated list of glob patterns, e.g. 'up,http_*')")
excludeMetrics = flag.String("exclude_metrics", os.Getenv("EXCLUDE_METRICS"), "Never publish the specified metrics (comma-separated list of glob patterns, e.g. 'tomcat_*')")
excludeDimensionsForMetrics = flag.String("exclude_dimensions_for_metrics", os.Getenv("EXCLUDE_DIMENSIONS_FOR_METRICS"), "Dimensions to exclude for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;')")
includeDimensionsForMetrics = flag.String("include_dimensions_for_metrics", os.Getenv("INCLUDE_DIMENSIONS_FOR_METRICS"), "Only publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job_id')")
excludeDimensionsForMetrics = flag.String("exclude_dimensions_for_metrics", os.Getenv("EXCLUDE_DIMENSIONS_FOR_METRICS"), "Never publish the specified dimensions for metrics (semi-colon-separated key values of comma-separated dimensions of METRIC=dim1,dim2;, e.g. 'flink_jobmanager=job,host;zk_up=host,pod;')")
)
// kevValMustParse takes a string and exits with a message if it cannot parse as KEY=VALUE
@ -41,6 +43,37 @@ func keyValMustParse(str, message string) (string, string) {
return kv[0], kv[1]
}
// dimensionMatcherListMustParse takes a string and a flag name and exists with a message
// if it cannot parse as GLOB=dim1,dim2;GLOB2=dim3
func dimensionMatcherListMustParse(str, flag string) []MatcherWithStringSet {
var matcherList []MatcherWithStringSet
// split metric1=dim1,dim2;metric2=dim1
// into [
// metric1=dim1,dim2
// metric*=dim1
// ]
// then into [{ Matcher: "metric1": Set: [dim1, dim2] } , { Matcher: "metric_*": Set: [dim1] }]
for _, sublist := range strings.Split(str, ";") {
key, val := keyValMustParse(sublist, fmt.Sprintf("%s must be formatted as METRIC_NAME=DIM_LIST;...", flag))
metricPattern, err := glob.Compile(key)
if err != nil {
log.Fatal(fmt.Errorf("prometheus-to-cloudwatch: Error: %s contains invalid glob pattern in '%s': %s", flag, key, err))
}
dims := strings.Split(val, ",")
if len(dims) == 0 {
log.Fatalf("prometheus-to-cloudwatch: Error: %s was not given dimensions to exclude for metric '%s'", flag, key)
}
g := MatcherWithStringSet{
Matcher: metricPattern,
Set: stringSliceToSet(dims),
}
matcherList = append(matcherList, g)
}
return matcherList
}
// stringSliceToSet creates a "set" (a boolean map) from a slice of strings
func stringSliceToSet(slice []string) StringSet {
boolMap := make(StringSet, len(slice))
@ -52,7 +85,6 @@ func stringSliceToSet(slice []string) StringSet {
return boolMap
}
func main() {
flag.Parse()
@ -123,30 +155,12 @@ func main() {
var excludeDimensionsForMetricsList []MatcherWithStringSet
if *excludeDimensionsForMetrics != "" {
// split metric1=dim1,dim2;metric2=dim1
// into [
// metric1=dim1,dim2
// metric*=dim1
// ]
// then into [{ Matcher: "metric1": Set: [dim1, dim2] } , { Matcher: "metric_*": Set: [dim1] }]
for _, sublist := range strings.Split(*excludeDimensionsForMetrics, ";") {
key, val := keyValMustParse(sublist, "-exclude_dimensions_for_metrics must be formatted as METRIC_NAME=DIM_LIST;...")
excludeDimensionsForMetricsList = dimensionMatcherListMustParse(*excludeDimensionsForMetrics, "-exclude_dimensions_for_metrics")
}
metricPattern, err := glob.Compile(key)
if err != nil {
log.Fatal(fmt.Errorf("prometheus-to-cloudwatch: Error: -exclude_dimensions_for_metrics contains invalid glob pattern in '%s': %s", key, err))
}
dims := strings.Split(val, ",")
if len(dims) == 0 {
log.Fatalf("prometheus-to-cloudwatch: Error: -exclude_dimensions_for_metrics was not given dimensions to exclude for metric '%s'", key)
}
g := MatcherWithStringSet{
Matcher: metricPattern,
Set: stringSliceToSet(dims),
}
excludeDimensionsForMetricsList = append(excludeDimensionsForMetricsList, g)
}
var includeDimensionsForMetricsList []MatcherWithStringSet
if *includeDimensionsForMetrics != "" {
includeDimensionsForMetricsList = dimensionMatcherListMustParse(*includeDimensionsForMetrics, "-include_dimensions_for_metrics")
}
config := &Config{
@ -163,6 +177,7 @@ func main() {
IncludeMetrics: includeMetricsList,
ExcludeMetrics: excludeMetricsList,
ExcludeDimensionsForMetrics: excludeDimensionsForMetricsList,
IncludeDimensionsForMetrics: includeDimensionsForMetricsList,
}
if *prometheusScrapeInterval != "" {

72
prometheus_to_cloudwatch.go
View File

@ -7,6 +7,14 @@ import (
"crypto/tls"
"errors"
"fmt"
"io"
"log"
"math"
"mime"
"net/http"
"sort"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
@ -17,13 +25,6 @@ import (
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
"github.com/prometheus/common/model"
"io"
"log"
"math"
"mime"
"net/http"
"sort"
"time"
)
const (
@ -41,6 +42,20 @@ type MatcherWithStringSet struct {
Set StringSet
}
// getMatchingSet returns the first set that matches a string, or nil if there is no match
func getMatchingSet(matcherSets []MatcherWithStringSet, str string) StringSet {
if matcherSets == nil {
return nil
}
for _, matcherWithSet := range matcherSets {
if matcherWithSet.Matcher.Match(str) {
return matcherWithSet.Set
}
}
return nil
}
// Config defines configuration options
type Config struct {
// AWS access key Id with permissions to publish CloudWatch metrics
@ -85,6 +100,9 @@ type Config struct {
// Never publish the specified metrics (a list of glob patterns, e.g. ["tomcat_*"])
ExcludeMetrics []glob.Glob
// Only publish certain dimensions from the specified metrics
IncludeDimensionsForMetrics []MatcherWithStringSet
// Exclude certain dimensions from the specified metrics
ExcludeDimensionsForMetrics []MatcherWithStringSet
}
@ -102,10 +120,10 @@ type Bridge struct {
replaceDimensions map[string]string
includeMetrics []glob.Glob
excludeMetrics []glob.Glob
includeDimensionsForMetrics []MatcherWithStringSet
excludeDimensionsForMetrics []MatcherWithStringSet
}
// NewBridge initializes and returns a pointer to a Bridge using the
// supplied configuration, or an error if there is a problem with the configuration
func NewBridge(c *Config) (*Bridge, error) {
@ -128,6 +146,7 @@ func NewBridge(c *Config) (*Bridge, error) {
b.replaceDimensions = c.ReplaceDimensions
b.includeMetrics = c.IncludeMetrics
b.excludeMetrics = c.ExcludeMetrics
b.includeDimensionsForMetrics = c.IncludeDimensionsForMetrics
b.excludeDimensionsForMetrics = c.ExcludeDimensionsForMetrics
if c.CloudWatchPublishInterval > 0 {
@ -275,16 +294,6 @@ func (b *Bridge) shouldIgnoreMetric(metricName string) bool {
return true
}
// getDimensionsToExcludeSetForMetric gets the dimensions blacklist for a metric, or nil if there isn't one
func (b *Bridge) getDimensionsToExcludeSetForMetric(metricName string) StringSet {
for _, matcherWithSet := range b.excludeDimensionsForMetrics {
if matcherWithSet.Matcher.Match(metricName) {
return matcherWithSet.Set
}
}
return nil
}
func anyPatternMatches(patterns []glob.Glob, s string) bool {
for _, pattern := range patterns {
if pattern.Match(s) {
@ -369,6 +378,27 @@ func getName(m model.Metric) string {
return ""
}
// shouldIncludeDimension determines whether or not to keep this dimension when publishing to cloudwatch
// if an `includeSet` is specified, this will only return `true` for dimensions in that set
func shouldIncludeDimension(dimName model.LabelName, includeSet, excludeSet StringSet) bool {
if dimName == model.MetricNameLabel || dimName == cwHighResLabel || dimName == cwUnitLabel {
return false
}
dimNameStr := string(dimName)
// blacklist first
if excludeSet != nil && excludeSet[dimNameStr] {
return false
}
// if no whitelist, keep it as it passed the blacklist
if includeSet == nil {
return true
}
// otherwise, check the whitelist
return includeSet[dimNameStr]
}
// getDimensions returns up to 10 dimensions for the provided metric - one for each label (except the __name__ label)
// If a metric has more than 10 labels, it attempts to behave deterministically and returning the first 10 labels as dimensions
func getDimensions(m model.Metric, num int, b *Bridge) ([]*cloudwatch.Dimension, []*cloudwatch.Dimension) {
@ -380,10 +410,12 @@ func getDimensions(m model.Metric, num int, b *Bridge) ([]*cloudwatch.Dimension,
names := make([]string, 0, len(m))
excludeSet := b.getDimensionsToExcludeSetForMetric(string(m[model.MetricNameLabel]))
metricName := getName(m)
includeSet := getMatchingSet(b.includeDimensionsForMetrics, metricName)
excludeSet := getMatchingSet(b.excludeDimensionsForMetrics, metricName)
for dimName := range m {
if !(dimName == model.MetricNameLabel || dimName == cwHighResLabel || dimName == cwUnitLabel || (excludeSet != nil && excludeSet[string(dimName)])) {
if shouldIncludeDimension(dimName, includeSet, excludeSet) {
names = append(names, string(dimName))
}
}