diff --git a/CHANGELOG.md b/CHANGELOG.md index 534e7f045aa..fdd02f94af6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Grafana Mimir +* [CHANGE] Query-frontend: Add support for UTF-8 label and metric names in `/api/v1/cardinality/{label_values|label_values|active_series}` endpoints. #11848. +* [CHANGE] Querier: Add support for UTF-8 label and metric names in `label_join`, `label_replace` and `count_values` PromQL functions. #11848. * [CHANGE] Remove support for Redis as a cache backend. #12163 * [CHANGE] Memcached: Remove experimental `-.memcached.addresses-provider` flag to use alternate DNS service discovery backends. The more reliable backend introduced in 2.16.0 (#10895) is now the default. As a result of this change, DNS-based cache service discovery no longer supports search domains. #12175 * [FEATURE] Distributor: Add experimental `-distributor.otel-native-delta-ingestion` option to allow primitive delta metrics ingestion via the OTLP endpoint. #11631 diff --git a/Makefile b/Makefile index ae92cd78a43..07726872cf5 100644 --- a/Makefile +++ b/Makefile @@ -344,7 +344,7 @@ lint: check-makefiles faillint -paths "github.com/grafana/mimir/pkg/..." ./pkg/querier/api/... faillint -paths "github.com/grafana/mimir/pkg/..." ./pkg/util/math/... - # Ensure all errors are report as APIError + # Ensure all errors are reported as APIError faillint -paths "github.com/weaveworks/common/httpgrpc.{Errorf}=github.com/grafana/mimir/pkg/api/error.Newf" ./pkg/frontend/querymiddleware/... # errors.Cause() only work on errors wrapped by github.com/pkg/errors, while it doesn't work diff --git a/go.mod b/go.mod index 73cff470d74..5162423cf61 100644 --- a/go.mod +++ b/go.mod @@ -343,7 +343,7 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v1.8.2-0.20250725113505-6dd7af9abc56 +replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v1.8.2-0.20250725123259-c4bd4faba234 // Replace memberlist with our fork which includes some fixes that haven't been // merged upstream yet: @@ -370,7 +370,7 @@ replace github.com/opentracing-contrib/go-stdlib => github.com/grafana/opentraci replace github.com/opentracing-contrib/go-grpc => github.com/charleskorn/go-grpc v0.0.0-20231024023642-e9298576254f // Replacing prometheus/alertmanager with our fork. -replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250620093340-be61a673dee6 +replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250722103749-329f0c4df1ba // Use Mimir fork of prometheus/otlptranslator to allow for higher velocity of upstream development, // while allowing Mimir to move at a more conservative pace. diff --git a/go.sum b/go.sum index ba49f67276c..da525c9b889 100644 --- a/go.sum +++ b/go.sum @@ -571,14 +571,14 @@ github.com/grafana/memberlist v0.3.1-0.20250428154222-f7d51a6f6700 h1:0t7iOQ5ZkB github.com/grafana/memberlist v0.3.1-0.20250428154222-f7d51a6f6700/go.mod h1:Ri9p/tRShbjYnpNf4FFPXG7wxEGY4Nrcn6E7jrVa//4= github.com/grafana/mimir-otlptranslator v0.0.0-20250703083430-c31a9568ad96 h1:kq5zJVW9LyFOB5xCeQPTON2HNjwwEkefhegZXGIhQPk= github.com/grafana/mimir-otlptranslator v0.0.0-20250703083430-c31a9568ad96/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI= -github.com/grafana/mimir-prometheus v1.8.2-0.20250725113505-6dd7af9abc56 h1:HdQyhMJ+AkDUTlFAU7qIIkC3C2n0R/cV8T2S7JnbK7U= -github.com/grafana/mimir-prometheus v1.8.2-0.20250725113505-6dd7af9abc56/go.mod h1:bi1IiCulyFfPIsfFMaCqlggqiLO4PyqNwK/DiqTaYDI= +github.com/grafana/mimir-prometheus v1.8.2-0.20250725123259-c4bd4faba234 h1:hwME5D5GMogJkN9yobyKTEvOk/SzUFDnYNkUjLsg3ik= +github.com/grafana/mimir-prometheus v1.8.2-0.20250725123259-c4bd4faba234/go.mod h1:Pe/2vVv91zryCeOwLSjIFJFsw4Pvd2VNHbTUGu6kUls= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 h1:em1oddjXL8c1tL0iFdtVtPloq2hRPen2MJQKoAWpxu0= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8= github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls= -github.com/grafana/prometheus-alertmanager v0.25.1-0.20250620093340-be61a673dee6 h1:oJnbhG6ZNy10AjsgNeAtAKeGHogIGOMfAsBH6fYYa5M= -github.com/grafana/prometheus-alertmanager v0.25.1-0.20250620093340-be61a673dee6/go.mod h1:O/QP1BCm0HHIzbKvgMzqb5sSyH88rzkFk84F4TfJjBU= +github.com/grafana/prometheus-alertmanager v0.25.1-0.20250722103749-329f0c4df1ba h1:8u5N0btFygn+2S+B6Xs0HFfq4NJ0kJsX9UpIOlidDmQ= +github.com/grafana/prometheus-alertmanager v0.25.1-0.20250722103749-329f0c4df1ba/go.mod h1:O/QP1BCm0HHIzbKvgMzqb5sSyH88rzkFk84F4TfJjBU= github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/grafana/regexp v0.0.0-20240531075221-3685f1377d7b h1:oMAq12GxTpwo9jxbnG/M4F/HdpwbibTaVoxNA0NZprY= diff --git a/integration/kv_test.go b/integration/kv_test.go index cf9b725919e..2fb1151b029 100644 --- a/integration/kv_test.go +++ b/integration/kv_test.go @@ -9,7 +9,7 @@ package integration import ( "context" "errors" - "sort" + "slices" "sync" "testing" "time" @@ -40,7 +40,7 @@ func TestKVList(t *testing.T) { // Get list of keys and sort them keys, err := client.List(context.Background(), "") require.NoError(t, err, "could not list keys") - sort.Strings(keys) + slices.Sort(keys) require.Equal(t, keysToCreate, keys, "returned key paths did not match created paths") verifyClientMetricsHistogram(t, reg, "cortex_kv_request_duration_seconds", map[string]uint64{ diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index e3d124e9c54..c8c48f3e2d8 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -66,12 +66,6 @@ import ( var tracer = otel.Tracer("pkg/distributor") -func init() { - // Mimir doesn't support Prometheus' UTF-8 metric/label name scheme yet. - // nolint:staticcheck - model.NameValidationScheme = model.LegacyValidation -} - var ( // Validation errors. errInvalidTenantShardSize = errors.New("invalid tenant shard size, the value must be greater than or equal to zero") @@ -1098,15 +1092,18 @@ func (d *Distributor) prePushRelabelMiddleware(next PushFunc) PushFunc { return err } + dropLabels := d.limits.DropLabels(userID) + relabelConfigs := d.limits.MetricRelabelConfigs(userID) + var removeTsIndexes []int lb := labels.NewBuilder(labels.EmptyLabels()) for tsIdx := 0; tsIdx < len(req.Timeseries); tsIdx++ { ts := req.Timeseries[tsIdx] - if mrc := d.limits.MetricRelabelConfigs(userID); len(mrc) > 0 { + if len(relabelConfigs) > 0 { mimirpb.FromLabelAdaptersToBuilder(ts.Labels, lb) lb.Set(metaLabelTenantID, userID) - keep := relabel.ProcessBuilder(lb, mrc...) + keep := relabel.ProcessBuilder(lb, relabelConfigs...) if !keep { removeTsIndexes = append(removeTsIndexes, tsIdx) continue @@ -1115,7 +1112,7 @@ func (d *Distributor) prePushRelabelMiddleware(next PushFunc) PushFunc { req.Timeseries[tsIdx].SetLabels(mimirpb.FromBuilderToLabelAdapters(lb, ts.Labels)) } - for _, labelName := range d.limits.DropLabels(userID) { + for _, labelName := range dropLabels { req.Timeseries[tsIdx].RemoveLabel(labelName) } diff --git a/pkg/distributor/validate.go b/pkg/distributor/validate.go index 54f2d205f6c..5cbd5a767a0 100644 --- a/pkg/distributor/validate.go +++ b/pkg/distributor/validate.go @@ -16,6 +16,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" "github.com/grafana/mimir/pkg/costattribution" "github.com/grafana/mimir/pkg/mimirpb" @@ -390,6 +391,7 @@ type labelValidationConfig interface { MaxLabelNamesPerInfoSeries(userID string) int MaxLabelNameLength(userID string) int MaxLabelValueLength(userID string) int + ValidationScheme(userID string) model.ValidationScheme } func removeNonASCIIChars(in string) (out string) { @@ -421,7 +423,9 @@ func validateLabels(m *sampleValidationMetrics, cfg labelValidationConfig, userI return errors.New(noMetricNameMsgFormat) } - if !model.IsValidMetricName(model.LabelValue(unsafeMetricName)) { + validationScheme := cfg.ValidationScheme(userID) + + if !labels.IsValidMetricName(unsafeMetricName, validationScheme) { cat.IncrementDiscardedSamples(ls, 1, reasonInvalidMetricName, ts) m.invalidMetricName.WithLabelValues(userID, group).Inc() return fmt.Errorf(invalidMetricNameMsgFormat, removeNonASCIIChars(unsafeMetricName)) @@ -447,7 +451,7 @@ func validateLabels(m *sampleValidationMetrics, cfg labelValidationConfig, userI maxLabelValueLength := cfg.MaxLabelValueLength(userID) lastLabelName := "" for _, l := range ls { - if !skipLabelValidation && !model.LabelName(l.Name).IsValid() { + if !skipLabelValidation && !labels.IsValidLabelName(l.Name, validationScheme) { m.invalidLabel.WithLabelValues(userID, group).Inc() cat.IncrementDiscardedSamples(ls, 1, reasonInvalidLabel, ts) return fmt.Errorf(invalidLabelMsgFormat, l.Name, mimirpb.FromLabelAdaptersToString(ls)) diff --git a/pkg/distributor/validate_test.go b/pkg/distributor/validate_test.go index 9945cedf2c0..3db8aef3646 100644 --- a/pkg/distributor/validate_test.go +++ b/pkg/distributor/validate_test.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "net/http" + "strconv" "strings" "testing" "time" @@ -29,6 +30,7 @@ import ( "github.com/grafana/mimir/pkg/costattribution" catestutils "github.com/grafana/mimir/pkg/costattribution/testutils" "github.com/grafana/mimir/pkg/mimirpb" + "github.com/grafana/mimir/pkg/util/globalerror" "github.com/grafana/mimir/pkg/util/validation" ) @@ -37,6 +39,7 @@ type validateLabelsCfg struct { maxLabelNamesPerInfoSeries int maxLabelNameLength int maxLabelValueLength int + validationScheme model.ValidationScheme } func (v validateLabelsCfg) MaxLabelNamesPerSeries(_ string) int { @@ -55,6 +58,10 @@ func (v validateLabelsCfg) MaxLabelValueLength(_ string) int { return v.maxLabelValueLength } +func (v validateLabelsCfg) ValidationScheme(_ string) model.ValidationScheme { + return v.validationScheme +} + type validateMetadataCfg struct { enforceMetadataMetricName bool maxMetadataLength int @@ -86,35 +93,59 @@ func TestValidateLabels(t *testing.T) { require.NoError(t, err) cast := manager.SampleTracker(userID) - for _, c := range []struct { + validationSchemes := []model.ValidationScheme{ + model.LegacyValidation, + model.UTF8Validation, + } + + // alwaysErr ensures this error is returned for legacy and utf8 validation. + alwaysErr := func(err error) func(model.ValidationScheme) error { + return func(model.ValidationScheme) error { + return err + } + } + + // legacyErr ensures err is only returned when legacy validation scheme is used. + legacyErr := func(err error) func(model.ValidationScheme) error { + return func(scheme model.ValidationScheme) error { + if scheme == model.LegacyValidation { + return err + } + return nil + } + } + + testCases := []struct { metric model.Metric skipLabelNameValidation bool skipLabelCountValidation bool - err error + wantErr func(model.ValidationScheme) error }{ { metric: map[model.LabelName]model.LabelValue{"team": "a"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: errors.New(noMetricNameMsgFormat), + wantErr: alwaysErr(errors.New(noMetricNameMsgFormat)), }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: " ", "team": "a"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: fmt.Errorf(invalidMetricNameMsgFormat, " "), + wantErr: legacyErr(fmt.Errorf(invalidMetricNameMsgFormat, " ")), }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "metric_name_with_\xb0_invalid_utf8_\xb0", "team": "a"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: fmt.Errorf(invalidMetricNameMsgFormat, "metric_name_with__invalid_utf8_ (non-ascii characters removed)"), + wantErr: alwaysErr( + fmt.Errorf(invalidMetricNameMsgFormat, "metric_name_with__invalid_utf8_ (non-ascii characters removed)"), + ), }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "valid", "foo ": "bar", "team": "a"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: fmt.Errorf( + wantErr: legacyErr(fmt.Errorf( invalidLabelMsgFormat, "foo ", mimirpb.FromLabelAdaptersToString( @@ -124,19 +155,18 @@ func TestValidateLabels(t *testing.T) { {Name: "team", Value: "a"}, }, ), - ), + )), }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "valid", "team": "c"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: nil, }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "badLabelName", "this_is_a_really_really_long_name_that_should_cause_an_error": "test_value_please_ignore", "team": "biz"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: fmt.Errorf( + wantErr: alwaysErr(fmt.Errorf( labelNameTooLongMsgFormat, "this_is_a_really_really_long_name_that_should_cause_an_error", mimirpb.FromLabelAdaptersToString( @@ -146,13 +176,13 @@ func TestValidateLabels(t *testing.T) { {Name: "this_is_a_really_really_long_name_that_should_cause_an_error", Value: "test_value_please_ignore"}, }, ), - ), + )), }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "badLabelValue", "much_shorter_name": "test_value_please_ignore_no_really_nothing_to_see_here", "team": "biz"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: LabelValueTooLongError{ + wantErr: alwaysErr(LabelValueTooLongError{ Label: mimirpb.LabelAdapter{Name: "much_shorter_name", Value: "test_value_please_ignore_no_really_nothing_to_see_here"}, Limit: 25, Series: []mimirpb.LabelAdapter{ @@ -160,13 +190,13 @@ func TestValidateLabels(t *testing.T) { {Name: "much_shorter_name", Value: "test_value_please_ignore_no_really_nothing_to_see_here"}, {Name: "team", Value: "biz"}, }, - }, + }), }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "bar": "baz", "blip": "blop", "team": "plof"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: fmt.Errorf( + wantErr: alwaysErr(fmt.Errorf( tooManyLabelsMsgFormat, tooManyLabelsArgs( []mimirpb.LabelAdapter{ @@ -177,21 +207,20 @@ func TestValidateLabels(t *testing.T) { }, 3, )..., - ), + )), }, { // *_info metrics have higher label limits. metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo_info", "bar": "baz", "blip": "blop", "team": "a"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: nil, }, { // *_info metrics have higher label limits. metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo_info", "bar": "baz", "blip": "blop", "blap": "blup", "team": "a"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: fmt.Errorf( + wantErr: alwaysErr(fmt.Errorf( tooManyInfoLabelsMsgFormat, tooManyLabelsArgs( []mimirpb.LabelAdapter{ @@ -203,83 +232,149 @@ func TestValidateLabels(t *testing.T) { }, 4, )..., - ), + )), }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "bar": "baz", "blip": "blop", "team": "a"}, skipLabelNameValidation: false, skipLabelCountValidation: true, - err: nil, }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "invalid%label&name": "bar", "team": "biz"}, skipLabelNameValidation: true, skipLabelCountValidation: false, - err: nil, }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "label1": "你好", "team": "plof"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: nil, }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "label1": "abc\xfe\xfddef", "team": "plof"}, skipLabelNameValidation: false, skipLabelCountValidation: false, - err: fmt.Errorf( + wantErr: alwaysErr(fmt.Errorf( invalidLabelValueMsgFormat, "label1", "abc\ufffddef", "foo", - ), + )), }, { metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "label1": "abc\xfe\xfddef"}, skipLabelNameValidation: true, skipLabelCountValidation: false, - err: nil, }, - } { - err := validateLabels(s, cfg, userID, "custom label", mimirpb.FromMetricsToLabelAdapters(c.metric), c.skipLabelNameValidation, c.skipLabelCountValidation, cast, ts) - assert.Equal(t, c.err, err, "wrong error") + { + metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "foo", "name😀": "value", "team": "b"}, + skipLabelNameValidation: false, + skipLabelCountValidation: false, + wantErr: legacyErr(fmt.Errorf( + invalidLabelMsgFormat, + "name😀", + mimirpb.FromLabelAdaptersToString( + []mimirpb.LabelAdapter{ + {Name: model.MetricNameLabel, Value: "foo"}, + {Name: "name😀", Value: "value"}, + {Name: "team", Value: "b"}, + }, + ), + )), + }, + { + metric: map[model.LabelName]model.LabelValue{model.MetricNameLabel: "name😀", "team": "b"}, + skipLabelNameValidation: false, + skipLabelCountValidation: false, + wantErr: legacyErr(fmt.Errorf( + invalidMetricNameMsgFormat, "name (non-ascii characters removed)"), + ), + }, + } + + for i, c := range testCases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + for _, scheme := range validationSchemes { + t.Run(scheme.String(), func(t *testing.T) { + testConfig := cfg + testConfig.validationScheme = scheme + var wantErr error + if c.wantErr != nil { + wantErr = c.wantErr(scheme) + } + err := validateLabels(s, testConfig, userID, "custom label", mimirpb.FromMetricsToLabelAdapters(c.metric), c.skipLabelNameValidation, c.skipLabelCountValidation, cast, ts) + assert.Equal(t, wantErr, err, "wrong error") + }) + } + }) + } + + discardedSamplesValues := map[string]map[string]int{} + for _, c := range testCases { + if c.wantErr == nil { + continue + } + for _, scheme := range validationSchemes { + if err := c.wantErr(scheme); err != nil { + for _, id := range []globalerror.ID{ + globalerror.SeriesInvalidLabel, + globalerror.SeriesInvalidLabelValue, + globalerror.SeriesLabelNameTooLong, + globalerror.SeriesLabelValueTooLong, + globalerror.MaxLabelNamesPerSeries, + globalerror.MaxLabelNamesPerInfoSeries, + globalerror.InvalidMetricName, + globalerror.MissingMetricName, + } { + if strings.Contains(err.Error(), string(id)) { + if discardedSamplesValues[id.LabelValue()] == nil { + discardedSamplesValues[id.LabelValue()] = map[string]int{} + } + team := string(c.metric["team"]) + discardedSamplesValues[id.LabelValue()][team]++ + } + } + } + } } randomReason := validation.DiscardedSamplesCounter(reg, "random reason") randomReason.WithLabelValues("different user", "custom label").Inc() - require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_discarded_samples_total The total number of samples that were discarded. - # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{group="custom label",reason="label_invalid",user="testUser"} 1 - cortex_discarded_samples_total{group="custom label",reason="label_name_too_long",user="testUser"} 1 - cortex_discarded_samples_total{group="custom label",reason="label_value_invalid",user="testUser"} 1 - cortex_discarded_samples_total{group="custom label",reason="label_value_too_long",user="testUser"} 1 - cortex_discarded_samples_total{group="custom label",reason="max_label_names_per_series",user="testUser"} 1 - cortex_discarded_samples_total{group="custom label",reason="max_label_names_per_info_series",user="testUser"} 1 - cortex_discarded_samples_total{group="custom label",reason="metric_name_invalid",user="testUser"} 2 - cortex_discarded_samples_total{group="custom label",reason="missing_metric_name",user="testUser"} 1 - cortex_discarded_samples_total{group="custom label",reason="random reason",user="different user"} 1 - `), "cortex_discarded_samples_total")) + wantDiscardedSamples := ` + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{group="custom label",reason="random reason",user="different user"} 1 + ` + wantDiscardedAttrSamples := ` + # HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution. + # TYPE cortex_discarded_attributed_samples_total counter + ` + + sumSamples := func(m map[string]int) (sum int) { + for _, v := range m { + sum += v + } + return + } - require.NoError(t, testutil.GatherAndCompare(careg, strings.NewReader(` - # HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution. - # TYPE cortex_discarded_attributed_samples_total counter - cortex_discarded_attributed_samples_total{reason="label_invalid",team="a",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="label_name_too_long",team="biz",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="label_value_invalid",team="plof",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="label_value_too_long",team="biz",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="max_label_names_per_info_series",team="a",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="max_label_names_per_series",team="plof",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="metric_name_invalid",team="a",tenant="testUser",tracker="cost-attribution"} 2 - cortex_discarded_attributed_samples_total{reason="missing_metric_name",team="a",tenant="testUser",tracker="cost-attribution"} 1 -`), "cortex_discarded_attributed_samples_total")) + for reason, countByTeam := range discardedSamplesValues { + wantDiscardedSamples += fmt.Sprintf( + `cortex_discarded_samples_total{group="custom label",reason="%s",user="testUser"} %d`+"\n", + reason, + sumSamples(countByTeam), + ) + for team, count := range countByTeam { + wantDiscardedAttrSamples += fmt.Sprintf( + `cortex_discarded_attributed_samples_total{reason="%s",team="%s",tenant="testUser",tracker="cost-attribution"} %d`+"\n", + reason, + team, + count, + ) + } + } - s.deleteUserMetrics(userID) + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(wantDiscardedSamples), "cortex_discarded_samples_total")) + require.NoError(t, testutil.GatherAndCompare(careg, strings.NewReader(wantDiscardedAttrSamples), "cortex_discarded_attributed_samples_total")) - require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_discarded_samples_total The total number of samples that were discarded. - # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{group="custom label",reason="random reason",user="different user"} 1 - `), "cortex_discarded_samples_total")) + s.deleteUserMetrics(userID) } func TestValidateExemplars(t *testing.T) { @@ -451,6 +546,7 @@ func TestValidateLabelDuplication(t *testing.T) { cfg.maxLabelNameLength = 10 cfg.maxLabelNamesPerSeries = 10 cfg.maxLabelValueLength = 10 + cfg.validationScheme = model.LegacyValidation userID := "testUser" actual := validateLabels(newSampleValidationMetrics(nil), cfg, userID, "", []mimirpb.LabelAdapter{ diff --git a/pkg/frontend/querymiddleware/request_validation.go b/pkg/frontend/querymiddleware/request_validation.go index 6db11683dc0..3d05c5f2f74 100644 --- a/pkg/frontend/querymiddleware/request_validation.go +++ b/pkg/frontend/querymiddleware/request_validation.go @@ -7,15 +7,8 @@ import ( "net/http" "github.com/grafana/dskit/cancellation" - "github.com/prometheus/common/model" ) -func init() { - // Mimir doesn't support Prometheus' UTF-8 metric/label name scheme yet. - // nolint:staticcheck - model.NameValidationScheme = model.LegacyValidation -} - const requestValidationFailedFmt = "request validation failed for " var errMetricsQueryRequestValidationFailed = cancellation.NewErrorf( diff --git a/pkg/frontend/querymiddleware/request_validation_test.go b/pkg/frontend/querymiddleware/request_validation_test.go index 7d32570f0d5..774289b3322 100644 --- a/pkg/frontend/querymiddleware/request_validation_test.go +++ b/pkg/frontend/querymiddleware/request_validation_test.go @@ -70,6 +70,21 @@ func TestMetricsQueryRequestValidationRoundTripper(t *testing.T) { url: instantQueryPathSuffix + "?query=up&start=123&end=456&step=60s", expectedErrType: "", }, + { + // accepts utf-8 label names + url: instantQueryPathSuffix + `?query=up{"test.label"="test"}`, + expectedErrType: "", + }, + { + // accepts utf-8 metric name + url: instantQueryPathSuffix + `?query={"test.label"}`, + expectedErrType: "", + }, + { + // invalid utf-8 string + url: instantQueryPathSuffix + "?query=up{\"test.label\"=\"\xff\"}", + expectedErrType: apierror.TypeBadData, + }, } { t.Run(strconv.Itoa(i), func(t *testing.T) { req, err := http.NewRequest(http.MethodGet, srv.URL+tc.url, nil) @@ -209,9 +224,13 @@ func TestCardinalityQueryRequestValidationRoundTripper(t *testing.T) { }, { // non-utf8 label name will be rejected even when we transition to UTF-8 label names - url: cardinalityLabelValuesPathSuffix + "?label_names[]=\\xbd\\xb2\\x3d\\xbc\\x20\\xe2\\x8c\\x98", + url: cardinalityLabelValuesPathSuffix + "?label_names[]=\xbd\xb2\x3d\xbc\x20\xe2\x8c\x98", expectedErrType: apierror.TypeBadData, }, + { + url: cardinalityLabelValuesPathSuffix + "?label_names[]=some.label", + expectedErrType: "", + }, { url: cardinalityLabelValuesPathSuffix + "?label_names[]=foo", expectedErrType: "", diff --git a/pkg/mimir/promexts.go b/pkg/mimir/promexts.go index cabc1122543..79356aaf6d9 100644 --- a/pkg/mimir/promexts.go +++ b/pkg/mimir/promexts.go @@ -3,14 +3,9 @@ package mimir import ( - "github.com/prometheus/common/model" - "github.com/grafana/mimir/pkg/util/promqlext" ) func init() { promqlext.ExtendPromQL() - // Mimir doesn't support Prometheus' UTF-8 metric/label name scheme yet. - // nolint:staticcheck - model.NameValidationScheme = model.LegacyValidation } diff --git a/pkg/mimirtool/commands/analyse_rulefiles.go b/pkg/mimirtool/commands/analyse_rulefiles.go index b75d24b474f..aafd62aae69 100644 --- a/pkg/mimirtool/commands/analyse_rulefiles.go +++ b/pkg/mimirtool/commands/analyse_rulefiles.go @@ -22,8 +22,8 @@ type RuleFileAnalyzeCommand struct { } func (cmd *RuleFileAnalyzeCommand) run(_ *kingpin.ParseContext) error { - - output, err := AnalyzeRuleFiles(cmd.RuleFilesList) + // TODO: Get scheme from CLI flag. + output, err := AnalyzeRuleFiles(cmd.RuleFilesList, model.LegacyValidation) if err != nil { return err } @@ -37,11 +37,11 @@ func (cmd *RuleFileAnalyzeCommand) run(_ *kingpin.ParseContext) error { } // AnalyzeRuleFiles analyze rules files and return the list metrics used in them. -func AnalyzeRuleFiles(ruleFiles []string) (*analyze.MetricsInRuler, error) { +func AnalyzeRuleFiles(ruleFiles []string, scheme model.ValidationScheme) (*analyze.MetricsInRuler, error) { output := &analyze.MetricsInRuler{} output.OverallMetrics = make(map[string]struct{}) - nss, err := rules.ParseFiles(rules.MimirBackend, ruleFiles) + nss, err := rules.ParseFiles(rules.MimirBackend, ruleFiles, scheme) if err != nil { return nil, errors.Wrap(err, "analyze operation unsuccessful, unable to parse rules files") } diff --git a/pkg/mimirtool/commands/analyse_rulefiles_test.go b/pkg/mimirtool/commands/analyse_rulefiles_test.go index fdf9d4edd3e..7a3948ce989 100644 --- a/pkg/mimirtool/commands/analyse_rulefiles_test.go +++ b/pkg/mimirtool/commands/analyse_rulefiles_test.go @@ -10,7 +10,7 @@ import ( ) func TestAnalyzeRuleFiles(t *testing.T) { - mir, err := AnalyzeRuleFiles([]string{"testdata/prometheus_rules.yaml"}) + mir, err := AnalyzeRuleFiles([]string{"testdata/prometheus_rules.yaml"}, model.UTF8Validation) require.NoError(t, err) require.Equal(t, 28, len(mir.MetricsUsed)) expectedMetrics := model.LabelValues{ diff --git a/pkg/mimirtool/commands/analyse_rules_test.go b/pkg/mimirtool/commands/analyse_rules_test.go index 6dda7489232..de345b99097 100644 --- a/pkg/mimirtool/commands/analyse_rules_test.go +++ b/pkg/mimirtool/commands/analyse_rules_test.go @@ -9,6 +9,7 @@ import ( "slices" "testing" + "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -56,7 +57,7 @@ func TestParseMetricsInRuleFile(t *testing.T) { output := &analyze.MetricsInRuler{} output.OverallMetrics = make(map[string]struct{}) - nss, err := rules.ParseFiles("mimir", []string{"testdata/prometheus_rules.yaml"}) + nss, err := rules.ParseFiles("mimir", []string{"testdata/prometheus_rules.yaml"}, model.UTF8Validation) require.NoError(t, err) for _, ns := range nss { diff --git a/pkg/mimirtool/commands/rules.go b/pkg/mimirtool/commands/rules.go index c55c3b99695..2b85a714676 100644 --- a/pkg/mimirtool/commands/rules.go +++ b/pkg/mimirtool/commands/rules.go @@ -19,6 +19,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/rulefmt" "github.com/prometheus/prometheus/promql/parser" log "github.com/sirupsen/logrus" @@ -475,7 +476,8 @@ func (r *RuleCommand) deleteRuleGroup(_ *kingpin.ParseContext) error { } func (r *RuleCommand) loadRules(_ *kingpin.ParseContext) error { - nss, err := rules.ParseFiles(r.Backend, r.RuleFilesList) + // TODO: Get scheme from CLI flag. + nss, err := rules.ParseFiles(r.Backend, r.RuleFilesList, model.LegacyValidation) if err != nil { return errors.Wrap(err, "load operation unsuccessful, unable to parse rules files") } @@ -544,7 +546,8 @@ func (r *RuleCommand) diffRules(_ *kingpin.ParseContext) error { return errors.Wrap(err, "diff operation unsuccessful, invalid arguments") } - nss, err := rules.ParseFiles(r.Backend, r.RuleFilesList) + // TODO: Get scheme from CLI flag. + nss, err := rules.ParseFiles(r.Backend, r.RuleFilesList, model.LegacyValidation) if err != nil { return errors.Wrap(err, "diff operation unsuccessful, unable to parse rules files") } @@ -612,7 +615,8 @@ func (r *RuleCommand) syncRules(_ *kingpin.ParseContext) error { return errors.Wrap(err, "sync operation unsuccessful, invalid arguments") } - nss, err := rules.ParseFiles(r.Backend, r.RuleFilesList) + // TODO: Get scheme from CLI flag. + nss, err := rules.ParseFiles(r.Backend, r.RuleFilesList, model.LegacyValidation) if err != nil { return errors.Wrap(err, "sync operation unsuccessful, unable to parse rules files") } @@ -720,7 +724,8 @@ func (r *RuleCommand) prepare(_ *kingpin.ParseContext) error { return errors.Wrap(err, "prepare operation unsuccessful, invalid arguments") } - namespaces, err := rules.ParseFiles(r.Backend, r.RuleFilesList) + // TODO: Get scheme from CLI flag. + namespaces, err := rules.ParseFiles(r.Backend, r.RuleFilesList, model.LegacyValidation) if err != nil { return errors.Wrap(err, "prepare operation unsuccessful, unable to parse rules files") } @@ -758,7 +763,8 @@ func (r *RuleCommand) lint(_ *kingpin.ParseContext) error { return errors.Wrap(err, "prepare operation unsuccessful, invalid arguments") } - namespaces, err := rules.ParseFiles(r.Backend, r.RuleFilesList) + // TODO: Get scheme from CLI flag. + namespaces, err := rules.ParseFiles(r.Backend, r.RuleFilesList, model.LegacyValidation) if err != nil { return errors.Wrap(err, "prepare operation unsuccessful, unable to parse rules files") } @@ -792,7 +798,8 @@ func (r *RuleCommand) checkRules(_ *kingpin.ParseContext) error { return errors.Wrap(err, "check operation unsuccessful, invalid arguments") } - namespaces, err := rules.ParseFiles(r.Backend, r.RuleFilesList) + // TODO: Get scheme from CLI flag. + namespaces, err := rules.ParseFiles(r.Backend, r.RuleFilesList, model.LegacyValidation) if err != nil { return errors.Wrap(err, "check operation unsuccessful, unable to parse rules files") } diff --git a/pkg/mimirtool/rules/parser.go b/pkg/mimirtool/rules/parser.go index 9263328a39c..01b84148406 100644 --- a/pkg/mimirtool/rules/parser.go +++ b/pkg/mimirtool/rules/parser.go @@ -14,6 +14,7 @@ import ( "path/filepath" "strings" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/rulefmt" log "github.com/sirupsen/logrus" yaml "gopkg.in/yaml.v3" @@ -29,9 +30,9 @@ var ( ) // ParseFiles returns a formatted set of prometheus rule groups -func ParseFiles(backend string, files []string) (map[string]RuleNamespace, error) { +func ParseFiles(backend string, files []string, scheme model.ValidationScheme) (map[string]RuleNamespace, error) { ruleSet := map[string]RuleNamespace{} - var parseFn func(f string) ([]RuleNamespace, []error) + var parseFn func(f string, scheme model.ValidationScheme) ([]RuleNamespace, []error) switch backend { case MimirBackend: parseFn = Parse @@ -40,7 +41,7 @@ func ParseFiles(backend string, files []string) (map[string]RuleNamespace, error } for _, f := range files { - nss, errs := parseFn(f) + nss, errs := parseFn(f, scheme) for _, err := range errs { log.WithError(err).WithField("file", f).Errorln("unable to parse rules file") return nil, errFileReadError @@ -72,17 +73,17 @@ func ParseFiles(backend string, files []string) (map[string]RuleNamespace, error } // Parse parses and validates a set of rules. -func Parse(f string) ([]RuleNamespace, []error) { +func Parse(f string, scheme model.ValidationScheme) ([]RuleNamespace, []error) { content, err := loadFile(f) if err != nil { log.WithError(err).WithField("file", f).Errorln("unable to load rules file") return nil, []error{errFileReadError} } - return ParseBytes(content) + return ParseBytes(content, scheme) } -func ParseBytes(content []byte) ([]RuleNamespace, []error) { +func ParseBytes(content []byte, scheme model.ValidationScheme) ([]RuleNamespace, []error) { decoder := yaml.NewDecoder(bytes.NewReader(content)) decoder.KnownFields(true) @@ -117,7 +118,7 @@ func ParseBytes(content []byte) ([]RuleNamespace, []error) { return nil, []error{err} } - if errs := ns.Validate(node.GroupNodes); len(errs) > 0 { + if errs := ns.Validate(node.GroupNodes, scheme); len(errs) > 0 { return nil, errs } } diff --git a/pkg/mimirtool/rules/parser_test.go b/pkg/mimirtool/rules/parser_test.go index f8aa220f87a..cceb04d455e 100644 --- a/pkg/mimirtool/rules/parser_test.go +++ b/pkg/mimirtool/rules/parser_test.go @@ -9,6 +9,7 @@ import ( "fmt" "testing" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/rulefmt" "github.com/grafana/mimir/pkg/mimirtool/rules/rwrulefmt" @@ -123,7 +124,7 @@ func TestParseFiles(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := ParseFiles(tt.backend, tt.files) + got, err := ParseFiles(tt.backend, tt.files, model.UTF8Validation) if (err != nil) != tt.wantErr { t.Errorf("ParseFiles() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/mimirtool/rules/rules.go b/pkg/mimirtool/rules/rules.go index 71de931eb76..1a68fab9576 100644 --- a/pkg/mimirtool/rules/rules.go +++ b/pkg/mimirtool/rules/rules.go @@ -9,6 +9,7 @@ import ( "fmt" "strings" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/rulefmt" "github.com/prometheus/prometheus/promql/parser" log "github.com/sirupsen/logrus" @@ -227,7 +228,7 @@ func prepareBinaryExpr(e *parser.BinaryExpr, label string, rule string) error { } // Validate each rule in the rule namespace is valid -func (r RuleNamespace) Validate(groupNodes []rulefmt.RuleGroupNode) []error { +func (r RuleNamespace) Validate(groupNodes []rulefmt.RuleGroupNode, scheme model.ValidationScheme) []error { set := map[string]struct{}{} var errs []error @@ -245,17 +246,17 @@ func (r RuleNamespace) Validate(groupNodes []rulefmt.RuleGroupNode) []error { set[g.Name] = struct{}{} - errs = append(errs, ValidateRuleGroup(g, groupNodes[i])...) + errs = append(errs, ValidateRuleGroup(g, groupNodes[i], scheme)...) } return errs } // ValidateRuleGroup validates a rulegroup -func ValidateRuleGroup(g rwrulefmt.RuleGroup, node rulefmt.RuleGroupNode) []error { +func ValidateRuleGroup(g rwrulefmt.RuleGroup, node rulefmt.RuleGroupNode, scheme model.ValidationScheme) []error { var errs []error for i, r := range g.Rules { - for _, err := range r.Validate(node.Rules[i]) { + for _, err := range r.Validate(node.Rules[i], scheme) { var ruleName string if r.Alert != "" { ruleName = r.Alert diff --git a/pkg/querier/cardinality_analysis_handler_test.go b/pkg/querier/cardinality_analysis_handler_test.go index 433cad44f03..9f41af82ca7 100644 --- a/pkg/querier/cardinality_analysis_handler_test.go +++ b/pkg/querier/cardinality_analysis_handler_test.go @@ -706,8 +706,8 @@ func TestLabelValuesCardinalityHandler_ParseError(t *testing.T) { expectedErrorMessage: "'label_names[]' param is required", }, "label_names param is invalid": { - url: "/label_values?label_names[]=olá", - expectedErrorMessage: "invalid 'label_names' param 'olá'", + url: "/label_values?label_names[]=\xff\xfe", + expectedErrorMessage: "invalid 'label_names' param '\xff\xfe'", }, "multiple selector params are provided": { url: "/label_values?label_names[]=hello&selector=foo&selector=bar", diff --git a/pkg/querier/stats_renderer_test.go b/pkg/querier/stats_renderer_test.go index bc9b12f3716..761f13dda06 100644 --- a/pkg/querier/stats_renderer_test.go +++ b/pkg/querier/stats_renderer_test.go @@ -13,7 +13,6 @@ import ( "github.com/grafana/dskit/user" "github.com/grafana/regexp" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/prometheus/common/route" "github.com/prometheus/prometheus/config" @@ -26,12 +25,6 @@ import ( mimir_stats "github.com/grafana/mimir/pkg/querier/stats" ) -func init() { - // Mimir doesn't support Prometheus' UTF-8 metric/label name scheme yet. - // nolint:staticcheck - model.NameValidationScheme = model.LegacyValidation -} - func TestStatsRenderer(t *testing.T) { testCases := map[string]struct { diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index fa87e200ab0..c515205ccc1 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -668,7 +668,7 @@ func (a *API) CreateRuleGroup(w http.ResponseWriter, req *http.Request) { return } - errs := a.ruler.manager.ValidateRuleGroup(rg, node) + errs := a.ruler.manager.ValidateRuleGroup(userID, rg, node) if len(errs) > 0 { e := []string{} for _, err := range errs { diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 3a001060e33..a18749f9369 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/dskit/user" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" @@ -216,6 +217,7 @@ type RulesLimits interface { RulerMaxIndependentRuleEvaluationConcurrencyPerTenant(userID string) int64 RulerAlertmanagerClientConfig(userID string) notifierCfg.AlertmanagerClientConfig RulerMinRuleEvaluationInterval(userID string) time.Duration + ValidationScheme(userID string) model.ValidationScheme } func MetricsQueryFunc(qf rules.QueryFunc, userID string, queries, failedQueries *prometheus.CounterVec, remoteQuerier bool) rules.QueryFunc { diff --git a/pkg/ruler/manager.go b/pkg/ruler/manager.go index d78a0a77b48..567a1834fab 100644 --- a/pkg/ruler/manager.go +++ b/pkg/ruler/manager.go @@ -416,7 +416,7 @@ func (r *DefaultMultiTenantManager) Stop() { r.mapper.cleanup() } -func (r *DefaultMultiTenantManager) ValidateRuleGroup(g rulefmt.RuleGroup, node rulefmt.RuleGroupNode) []error { +func (r *DefaultMultiTenantManager) ValidateRuleGroup(userID string, g rulefmt.RuleGroup, node rulefmt.RuleGroupNode) []error { var errs []error if g.Name == "" { @@ -439,8 +439,9 @@ func (r *DefaultMultiTenantManager) ValidateRuleGroup(g rulefmt.RuleGroup, node errs = append(errs, fmt.Errorf("invalid rules configuration: rule group '%s' has both query_offset and (deprecated) evaluation_delay set, but to different values; please remove the deprecated evaluation_delay and use query_offset instead", g.Name)) } + validationScheme := r.limits.ValidationScheme(userID) for i, r := range g.Rules { - for _, err := range r.Validate(node.Rules[i]) { + for _, err := range r.Validate(node.Rules[i], validationScheme) { var ruleName string if r.Alert != "" { ruleName = r.Alert diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 13e7346659e..ce77431702f 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -291,7 +291,7 @@ type MultiTenantManager interface { Stop() // ValidateRuleGroup validates a rulegroup - ValidateRuleGroup(rulefmt.RuleGroup, rulefmt.RuleGroupNode) []error + ValidateRuleGroup(userID string, ruleGroup rulefmt.RuleGroup, ruleGroupNode rulefmt.RuleGroupNode) []error // Start evaluating rules. Start() @@ -1353,6 +1353,11 @@ func (r *Ruler) IsMaxRuleGroupsLimited(userID, namespace string) bool { return r.limits.RulerMaxRuleGroupsPerTenant(userID, namespace) > 0 } +// NameValidationScheme returns the validation scheme to use for a particular tenant. +func (r *Ruler) NameValidationScheme(userID string) model.ValidationScheme { + return r.limits.ValidationScheme(userID) +} + // AssertMaxRuleGroups limit has not been reached compared to the current // number of total rule groups in input and returns an error if so. func (r *Ruler) AssertMaxRuleGroups(userID, namespace string, rg int) error { diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 3df835d0351..ba540a0d180 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -561,6 +561,8 @@ func (l *Limits) validate() error { if cfg == nil { return errors.New("invalid metric_relabel_configs") } + // TODO: when we make validation scheme configurable, set + // cfg.MetricNameValidationScheme to match that value. } if l.MaxEstimatedChunksPerQueryMultiplier < 1 && l.MaxEstimatedChunksPerQueryMultiplier != 0 { @@ -1056,7 +1058,12 @@ func (o *Overrides) CompactorBlockUploadMaxBlockSizeBytes(userID string) int64 { // MetricRelabelConfigs returns the metric relabel configs for a given user. func (o *Overrides) MetricRelabelConfigs(userID string) []*relabel.Config { - return o.getOverridesForUser(userID).MetricRelabelConfigs + relabelConfigs := o.getOverridesForUser(userID).MetricRelabelConfigs + validationScheme := o.ValidationScheme(userID) + for i := range relabelConfigs { + relabelConfigs[i].MetricNameValidationScheme = validationScheme + } + return relabelConfigs } func (o *Overrides) MetricRelabelingEnabled(userID string) bool { @@ -1412,6 +1419,12 @@ func (o *Overrides) LabelsQueryOptimizerEnabled(userID string) bool { return o.getOverridesForUser(userID).LabelsQueryOptimizerEnabled } +// ValidationScheme returns the validation scheme to use for a particular tenant. +func (o *Overrides) ValidationScheme(_ string) model.ValidationScheme { + // TODO(juliusmh): make this configurable by tenant + return model.LegacyValidation +} + // CardinalityAnalysisMaxResults returns the maximum number of results that // can be returned in a single cardinality API request. func (o *Overrides) CardinalityAnalysisMaxResults(userID string) int { diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index a5c5072cd7b..bcbeeced255 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -163,6 +163,7 @@ metric_relabel_configs: require.NoError(t, err) exp.Regex = regex exp.SourceLabels = model.LabelNames([]model.LabelName{"le"}) + exp.MetricNameValidationScheme = model.UTF8Validation l := Limits{} dec := yaml.NewDecoder(strings.NewReader(inp)) diff --git a/vendor/github.com/prometheus/alertmanager/matchers/compat/parse.go b/vendor/github.com/prometheus/alertmanager/matchers/compat/parse.go index 0c0dfffb1fd..951310268c9 100644 --- a/vendor/github.com/prometheus/alertmanager/matchers/compat/parse.go +++ b/vendor/github.com/prometheus/alertmanager/matchers/compat/parse.go @@ -190,7 +190,7 @@ func FallbackMatchersParser(l log.Logger) ParseMatchers { // isValidClassicLabelName returns true if the string is a valid classic label name. func isValidClassicLabelName(_ log.Logger) func(model.LabelName) bool { return func(name model.LabelName) bool { - return name.IsValid() + return name.IsValidLegacy() } } diff --git a/vendor/github.com/prometheus/prometheus/model/labels/validate.go b/vendor/github.com/prometheus/prometheus/model/labels/validate.go new file mode 100644 index 00000000000..db56ed47eaa --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/model/labels/validate.go @@ -0,0 +1,32 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labels + +import "github.com/prometheus/common/model" + +// IsValidMetricName returns whether name is a valid metric name, depending on the validation scheme. +func IsValidMetricName(name string, scheme model.ValidationScheme) bool { + if scheme == model.LegacyValidation { + return model.IsValidLegacyMetricName(name) + } + return model.IsValidMetricName(model.LabelValue(name)) +} + +// IsValidLabelName returns whether name is a valid label name, depending on the validation scheme. +func IsValidLabelName(name string, scheme model.ValidationScheme) bool { + if scheme == model.LegacyValidation { + return model.LabelName(name).IsValidLegacy() + } + return model.LabelName(name).IsValid() +} diff --git a/vendor/github.com/prometheus/prometheus/model/relabel/relabel.go b/vendor/github.com/prometheus/prometheus/model/relabel/relabel.go index 70daef426f5..72b73de6340 100644 --- a/vendor/github.com/prometheus/prometheus/model/relabel/relabel.go +++ b/vendor/github.com/prometheus/prometheus/model/relabel/relabel.go @@ -100,6 +100,8 @@ type Config struct { Replacement string `yaml:"replacement,omitempty" json:"replacement,omitempty"` // Action is the action to be performed for the relabeling. Action Action `yaml:"action,omitempty" json:"action,omitempty"` + // MetricNameValidationScheme to use when validating labels. + MetricNameValidationScheme model.ValidationScheme `yaml:"-" json:"-"` } // UnmarshalYAML implements the yaml.Unmarshaler interface. @@ -112,6 +114,14 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { if c.Regex.Regexp == nil { c.Regex = MustNewRegexp("") } + switch c.MetricNameValidationScheme { + case model.LegacyValidation, model.UTF8Validation: + case model.UnsetValidation: + //nolint:staticcheck // model.NameValidationScheme is deprecated. + c.MetricNameValidationScheme = model.NameValidationScheme + default: + return fmt.Errorf("unknown global name validation method specified, must be either '', 'legacy' or 'utf8', got %s", c.MetricNameValidationScheme) + } return c.Validate() } @@ -125,7 +135,14 @@ func (c *Config) Validate() error { if (c.Action == Replace || c.Action == HashMod || c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) && c.TargetLabel == "" { return fmt.Errorf("relabel configuration for %s action requires 'target_label' value", c.Action) } - if c.Action == Replace && !varInRegexTemplate(c.TargetLabel) && !model.LabelName(c.TargetLabel).IsValid() { + + switch c.MetricNameValidationScheme { + case model.LegacyValidation, model.UTF8Validation: + default: + return errors.New("MetricNameValidationScheme must be set in relabel configuration") + } + + if c.Action == Replace && !varInRegexTemplate(c.TargetLabel) && !labels.IsValidLabelName(c.TargetLabel, c.MetricNameValidationScheme) { return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action) } @@ -133,12 +150,12 @@ func (c *Config) Validate() error { // UTF-8 allows ${} characters, so standard validation allow $variables by default. // TODO(bwplotka): Relabelling users cannot put $ and ${<...>} characters in metric names or values. // Design escaping mechanism to allow that, once valid use case appears. - return model.LabelName(value).IsValid() + return labels.IsValidLabelName(value, c.MetricNameValidationScheme) } if c.Action == Replace && varInRegexTemplate(c.TargetLabel) && !isValidLabelNameWithRegexVarFn(c.TargetLabel) { return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action) } - if (c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) && !model.LabelName(c.TargetLabel).IsValid() { + if (c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) && !labels.IsValidLabelName(c.TargetLabel, c.MetricNameValidationScheme) { return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action) } if (c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) && c.Replacement != DefaultRelabelConfig.Replacement { @@ -147,7 +164,7 @@ func (c *Config) Validate() error { if c.Action == LabelMap && !isValidLabelNameWithRegexVarFn(c.Replacement) { return fmt.Errorf("%q is invalid 'replacement' for %s action", c.Replacement, c.Action) } - if c.Action == HashMod && !model.LabelName(c.TargetLabel).IsValid() { + if c.Action == HashMod && !labels.IsValidLabelName(c.TargetLabel, c.MetricNameValidationScheme) { return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action) } @@ -318,16 +335,16 @@ func relabel(cfg *Config, lb *labels.Builder) (keep bool) { if indexes == nil { break } - target := model.LabelName(cfg.Regex.ExpandString([]byte{}, cfg.TargetLabel, val, indexes)) - if !target.IsValid() { + target := string(cfg.Regex.ExpandString([]byte{}, cfg.TargetLabel, val, indexes)) + if !labels.IsValidLabelName(target, cfg.MetricNameValidationScheme) { break } res := cfg.Regex.ExpandString([]byte{}, cfg.Replacement, val, indexes) if len(res) == 0 { - lb.Del(string(target)) + lb.Del(target) break } - lb.Set(string(target), string(res)) + lb.Set(target, string(res)) case Lowercase: lb.Set(cfg.TargetLabel, strings.ToLower(val)) case Uppercase: diff --git a/vendor/github.com/prometheus/prometheus/model/rulefmt/rulefmt.go b/vendor/github.com/prometheus/prometheus/model/rulefmt/rulefmt.go index 96b70cc66b0..4e9d78abd0b 100644 --- a/vendor/github.com/prometheus/prometheus/model/rulefmt/rulefmt.go +++ b/vendor/github.com/prometheus/prometheus/model/rulefmt/rulefmt.go @@ -26,6 +26,7 @@ import ( "github.com/prometheus/common/model" "gopkg.in/yaml.v3" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" @@ -96,7 +97,7 @@ type ruleGroups struct { } // Validate validates all rules in the rule groups. -func (g *RuleGroups) Validate(node ruleGroups) (errs []error) { +func (g *RuleGroups) Validate(node ruleGroups, validationScheme model.ValidationScheme) (errs []error) { set := map[string]struct{}{} for j, g := range g.Groups { @@ -112,7 +113,7 @@ func (g *RuleGroups) Validate(node ruleGroups) (errs []error) { } for k, v := range g.Labels { - if !model.LabelName(k).IsValid() || k == model.MetricNameLabel { + if !labels.IsValidLabelName(k, validationScheme) || k == model.MetricNameLabel { errs = append( errs, fmt.Errorf("invalid label name: %s", k), ) @@ -128,7 +129,7 @@ func (g *RuleGroups) Validate(node ruleGroups) (errs []error) { set[g.Name] = struct{}{} for i, r := range g.Rules { - for _, node := range r.Validate(node.Groups[j].Rules[i]) { + for _, node := range r.Validate(node.Groups[j].Rules[i], validationScheme) { var ruleName string if r.Alert != "" { ruleName = r.Alert @@ -198,7 +199,7 @@ type RuleNode struct { } // Validate the rule and return a list of encountered errors. -func (r *Rule) Validate(node RuleNode) (nodes []WrappedError) { +func (r *Rule) Validate(node RuleNode, validationScheme model.ValidationScheme) (nodes []WrappedError) { if r.Record != "" && r.Alert != "" { nodes = append(nodes, WrappedError{ err: errors.New("only one of 'record' and 'alert' must be set"), @@ -244,7 +245,7 @@ func (r *Rule) Validate(node RuleNode) (nodes []WrappedError) { node: &node.Record, }) } - if !model.IsValidMetricName(model.LabelValue(r.Record)) { + if !labels.IsValidMetricName(r.Record, validationScheme) { nodes = append(nodes, WrappedError{ err: fmt.Errorf("invalid recording rule name: %s", r.Record), node: &node.Record, @@ -261,7 +262,7 @@ func (r *Rule) Validate(node RuleNode) (nodes []WrappedError) { } for k, v := range r.Labels { - if !model.LabelName(k).IsValid() || k == model.MetricNameLabel { + if !labels.IsValidLabelName(k, validationScheme) || k == model.MetricNameLabel { nodes = append(nodes, WrappedError{ err: fmt.Errorf("invalid label name: %s", k), }) @@ -275,7 +276,7 @@ func (r *Rule) Validate(node RuleNode) (nodes []WrappedError) { } for k := range r.Annotations { - if !model.LabelName(k).IsValid() { + if !labels.IsValidLabelName(k, validationScheme) { nodes = append(nodes, WrappedError{ err: fmt.Errorf("invalid annotation name: %s", k), }) @@ -338,8 +339,38 @@ func testTemplateParsing(rl *Rule) (errs []error) { return errs } +type parseArgs struct { + validationScheme model.ValidationScheme + ignoreUnknownFields bool +} + +type ParseOption func(*parseArgs) + +// WithValidationScheme returns a ParseOption setting the metric/label name validation scheme. +func WithValidationScheme(scheme model.ValidationScheme) ParseOption { + return func(args *parseArgs) { + args.validationScheme = scheme + } +} + +// WithIgnoreUnknownFields returns a ParseOption setting whether to ignore unknown fields. +func WithIgnoreUnknownFields(ignoreUnknownFields bool) ParseOption { + return func(args *parseArgs) { + args.ignoreUnknownFields = ignoreUnknownFields + } +} + // Parse parses and validates a set of rules. -func Parse(content []byte, ignoreUnknownFields bool) (*RuleGroups, []error) { +// The default metric/label name validation scheme is model.NameValidationScheme. +func Parse(content []byte, opts ...ParseOption) (*RuleGroups, []error) { + args := &parseArgs{ + //nolint:staticcheck // model.NameValidationScheme is deprecated. + validationScheme: model.NameValidationScheme, + } + for _, opt := range opts { + opt(args) + } + var ( groups RuleGroups node ruleGroups @@ -347,7 +378,7 @@ func Parse(content []byte, ignoreUnknownFields bool) (*RuleGroups, []error) { ) decoder := yaml.NewDecoder(bytes.NewReader(content)) - if !ignoreUnknownFields { + if !args.ignoreUnknownFields { decoder.KnownFields(true) } err := decoder.Decode(&groups) @@ -364,16 +395,16 @@ func Parse(content []byte, ignoreUnknownFields bool) (*RuleGroups, []error) { return nil, errs } - return &groups, groups.Validate(node) + return &groups, groups.Validate(node, args.validationScheme) } // ParseFile reads and parses rules from a file. -func ParseFile(file string, ignoreUnknownFields bool) (*RuleGroups, []error) { +func ParseFile(file string, opts ...ParseOption) (*RuleGroups, []error) { b, err := os.ReadFile(file) if err != nil { return nil, []error{fmt.Errorf("%s: %w", file, err)} } - rgs, errs := Parse(b, ignoreUnknownFields) + rgs, errs := Parse(b, opts...) for i := range errs { errs[i] = fmt.Errorf("%s: %w", file, errs[i]) } diff --git a/vendor/github.com/prometheus/prometheus/notifier/manager.go b/vendor/github.com/prometheus/prometheus/notifier/manager.go index c9463b24a8d..a7fe43eb774 100644 --- a/vendor/github.com/prometheus/prometheus/notifier/manager.go +++ b/vendor/github.com/prometheus/prometheus/notifier/manager.go @@ -26,6 +26,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/prometheus/common/version" @@ -104,6 +105,20 @@ func NewManager(o *Options, logger *slog.Logger) *Manager { logger = promslog.NewNopLogger() } + for i, rc := range o.RelabelConfigs { + switch rc.MetricNameValidationScheme { + case model.LegacyValidation, model.UTF8Validation: + default: + //nolint:staticcheck // model.NameValidationScheme is deprecated. + o.RelabelConfigs[i].MetricNameValidationScheme = model.NameValidationScheme + logger.Warn( + "notifier.NewManager: using default metric/label name validation scheme", + "relabel_config", i, + "scheme", o.RelabelConfigs[i].MetricNameValidationScheme, + ) + } + } + n := &Manager{ queue: make([]*Alert, 0, o.QueueCapacity), more: make(chan struct{}, 1), @@ -133,6 +148,14 @@ func (n *Manager) ApplyConfig(conf *config.Config) error { n.opts.ExternalLabels = conf.GlobalConfig.ExternalLabels n.opts.RelabelConfigs = conf.AlertingConfig.AlertRelabelConfigs + for i, rc := range n.opts.RelabelConfigs { + switch rc.MetricNameValidationScheme { + case model.LegacyValidation, model.UTF8Validation: + default: + //nolint:staticcheck // model.NameValidationScheme is deprecated. + n.opts.RelabelConfigs[i].MetricNameValidationScheme = model.NameValidationScheme + } + } amSets := make(map[string]*alertmanagerSet) // configToAlertmanagers maps alertmanager sets for each unique AlertmanagerConfig, diff --git a/vendor/github.com/prometheus/prometheus/rules/manager.go b/vendor/github.com/prometheus/prometheus/rules/manager.go index 575a32ba280..6ecdf793d1e 100644 --- a/vendor/github.com/prometheus/prometheus/rules/manager.go +++ b/vendor/github.com/prometheus/prometheus/rules/manager.go @@ -309,7 +309,7 @@ type GroupLoader interface { type FileLoader struct{} func (FileLoader) Load(identifier string, ignoreUnknownFields bool) (*rulefmt.RuleGroups, []error) { - return rulefmt.ParseFile(identifier, ignoreUnknownFields) + return rulefmt.ParseFile(identifier, rulefmt.WithIgnoreUnknownFields(ignoreUnknownFields)) } func (FileLoader) Parse(query string) (parser.Expr, error) { return parser.ParseExpr(query) } @@ -627,7 +627,7 @@ func ParseFiles(patterns []string) error { } } for fn, pat := range files { - _, errs := rulefmt.ParseFile(fn, false) + _, errs := rulefmt.ParseFile(fn) if len(errs) > 0 { return fmt.Errorf("parse rules from file %q (pattern: %q): %w", fn, pat, errors.Join(errs...)) } diff --git a/vendor/github.com/prometheus/prometheus/scrape/scrape.go b/vendor/github.com/prometheus/prometheus/scrape/scrape.go index b4f34a6f5b3..b0f7047b9e0 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/scrape.go +++ b/vendor/github.com/prometheus/prometheus/scrape/scrape.go @@ -154,6 +154,11 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed return nil, err } + switch cfg.MetricNameValidationScheme { + case model.LegacyValidation, model.UTF8Validation: + default: + return nil, errors.New("cfg.MetricNameValidationScheme must be set in scrape configuration") + } var escapingScheme model.EscapingScheme escapingScheme, err = config.ToEscapingScheme(cfg.MetricNameEscapingScheme, cfg.MetricNameValidationScheme) if err != nil { @@ -326,6 +331,11 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { sp.config = cfg oldClient := sp.client sp.client = client + switch cfg.MetricNameValidationScheme { + case model.LegacyValidation, model.UTF8Validation: + default: + return errors.New("cfg.MetricNameValidationScheme must be set in scrape configuration") + } sp.validationScheme = cfg.MetricNameValidationScheme var escapingScheme model.EscapingScheme escapingScheme, err = model.ToEscapingScheme(cfg.MetricNameEscapingScheme) diff --git a/vendor/modules.txt b/vendor/modules.txt index c1af96cda5a..24e7bee1f40 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1099,7 +1099,7 @@ github.com/pmezard/go-difflib/difflib # github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c ## explicit; go 1.14 github.com/power-devops/perfstat -# github.com/prometheus/alertmanager v0.28.1 => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250620093340-be61a673dee6 +# github.com/prometheus/alertmanager v0.28.1 => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250722103749-329f0c4df1ba ## explicit; go 1.23.0 github.com/prometheus/alertmanager/api github.com/prometheus/alertmanager/api/metrics @@ -1192,7 +1192,7 @@ github.com/prometheus/otlptranslator github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v1.8.2-0.20250725113505-6dd7af9abc56 +# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v1.8.2-0.20250725123259-c4bd4faba234 ## explicit; go 1.23.0 github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery @@ -2112,13 +2112,13 @@ sigs.k8s.io/kustomize/kyaml/yaml/walk sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 sigs.k8s.io/yaml/goyaml.v3 -# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v1.8.2-0.20250725113505-6dd7af9abc56 +# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v1.8.2-0.20250725123259-c4bd4faba234 # github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20250428154222-f7d51a6f6700 # gopkg.in/yaml.v3 => github.com/colega/go-yaml-yaml v0.0.0-20220720105220-255a8d16d094 # github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20240531075221-3685f1377d7b # github.com/munnerz/goautoneg => github.com/grafana/goautoneg v0.0.0-20240607115440-f335c04c58ce # github.com/opentracing-contrib/go-stdlib => github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 # github.com/opentracing-contrib/go-grpc => github.com/charleskorn/go-grpc v0.0.0-20231024023642-e9298576254f -# github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250620093340-be61a673dee6 +# github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250722103749-329f0c4df1ba # github.com/prometheus/otlptranslator => github.com/grafana/mimir-otlptranslator v0.0.0-20250703083430-c31a9568ad96 # github.com/thanos-io/objstore => github.com/charleskorn/objstore v0.0.0-20250527065533-21d4c0c463eb