fix: Fix ruler panic related to unset validation scheme (backport release-3.7.x) (#21401)

Co-authored-by: Paul Rogers <129207811+paul1r@users.noreply.github.com>

Author: loki-gh-app[bot]

pkg/ruler/base/notifier.go

@@ -154,6 +154,9 @@ func buildNotifierConfig(amConfig *ruler_config.AlertManagerConfig, externalLabe
 	promConfig := &config.Config{
 		GlobalConfig: config.GlobalConfig{
 			ExternalLabels: externalLabels,
+			// Prometheus notifier ApplyConfig assigns this to alert relabel configs whose
+			// scheme is unset; model.UnsetValidation panics during notify (see grafana/loki#21368).
+			MetricNameValidationScheme: model.UTF8Validation,
 		},
 		AlertingConfig: config.AlertingConfig{
 			AlertRelabelConfigs: amConfig.AlertRelabelConfigs,

pkg/ruler/base/notifier_test.go

@@ -5,13 +5,16 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	config_util "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/dns"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
+	"github.com/prometheus/prometheus/notifier"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/dskit/flagext"
@@ -40,6 +43,9 @@ func TestBuildNotifierConfig(t *testing.T) {
 				},
 			},
 			ncfg: &config.Config{
+				GlobalConfig: config.GlobalConfig{
+					MetricNameValidationScheme: model.UTF8Validation,
+				},
 				AlertingConfig: config.AlertingConfig{
 					AlertmanagerConfigs: []*config.AlertmanagerConfig{
 						{
@@ -68,6 +74,9 @@ func TestBuildNotifierConfig(t *testing.T) {
 				},
 			},
 			ncfg: &config.Config{
+				GlobalConfig: config.GlobalConfig{
+					MetricNameValidationScheme: model.UTF8Validation,
+				},
 				AlertingConfig: config.AlertingConfig{
 					AlertmanagerConfigs: []*config.AlertmanagerConfig{
 						{
@@ -105,6 +114,9 @@ func TestBuildNotifierConfig(t *testing.T) {
 				},
 			},
 			ncfg: &config.Config{
+				GlobalConfig: config.GlobalConfig{
+					MetricNameValidationScheme: model.UTF8Validation,
+				},
 				AlertingConfig: config.AlertingConfig{
 					AlertmanagerConfigs: []*config.AlertmanagerConfig{
 						{
@@ -143,6 +155,9 @@ func TestBuildNotifierConfig(t *testing.T) {
 				},
 			},
 			ncfg: &config.Config{
+				GlobalConfig: config.GlobalConfig{
+					MetricNameValidationScheme: model.UTF8Validation,
+				},
 				AlertingConfig: config.AlertingConfig{
 					AlertmanagerConfigs: []*config.AlertmanagerConfig{
 						{
@@ -183,6 +198,9 @@ func TestBuildNotifierConfig(t *testing.T) {
 				},
 			},
 			ncfg: &config.Config{
+				GlobalConfig: config.GlobalConfig{
+					MetricNameValidationScheme: model.UTF8Validation,
+				},
 				AlertingConfig: config.AlertingConfig{
 					AlertmanagerConfigs: []*config.AlertmanagerConfig{
 						{
@@ -218,6 +236,9 @@ func TestBuildNotifierConfig(t *testing.T) {
 				},
 			},
 			ncfg: &config.Config{
+				GlobalConfig: config.GlobalConfig{
+					MetricNameValidationScheme: model.UTF8Validation,
+				},
 				AlertingConfig: config.AlertingConfig{
 					AlertmanagerConfigs: []*config.AlertmanagerConfig{
 						{
@@ -253,6 +274,9 @@ func TestBuildNotifierConfig(t *testing.T) {
 				},
 			},
 			ncfg: &config.Config{
+				GlobalConfig: config.GlobalConfig{
+					MetricNameValidationScheme: model.UTF8Validation,
+				},
 				AlertingConfig: config.AlertingConfig{
 					AlertmanagerConfigs: []*config.AlertmanagerConfig{
 						{
@@ -291,6 +315,9 @@ func TestBuildNotifierConfig(t *testing.T) {
 				},
 			},
 			ncfg: &config.Config{
+				GlobalConfig: config.GlobalConfig{
+					MetricNameValidationScheme: model.UTF8Validation,
+				},
 				AlertingConfig: config.AlertingConfig{
 					AlertmanagerConfigs: []*config.AlertmanagerConfig{
 						{
@@ -341,7 +368,8 @@ func TestBuildNotifierConfig(t *testing.T) {
 					},
 				},
 				GlobalConfig: config.GlobalConfig{
-					ExternalLabels: labels.FromStrings("region", "us-east-1"),
+					ExternalLabels:             labels.FromStrings("region", "us-east-1"),
+					MetricNameValidationScheme: model.UTF8Validation,
 				},
 			},
 		},
@@ -387,7 +415,8 @@ func TestBuildNotifierConfig(t *testing.T) {
 					},
 				},
 				GlobalConfig: config.GlobalConfig{
-					ExternalLabels: labels.FromStrings("region", "us-east-1"),
+					ExternalLabels:             labels.FromStrings("region", "us-east-1"),
+					MetricNameValidationScheme: model.UTF8Validation,
 				},
 			},
 		},
@@ -406,6 +435,49 @@ func TestBuildNotifierConfig(t *testing.T) {
 	}
 }
 
+// TestBuildNotifierConfig_SetsUTF8MetricNameValidationScheme fails if GlobalConfig leaves
+// MetricNameValidationScheme unset (model.UnsetValidation): notifier.ApplyConfig then copies
+// that onto alert_relabel_configs and alert delivery can panic (grafana/loki#21368).
+func TestBuildNotifierConfig_SetsUTF8MetricNameValidationScheme(t *testing.T) {
+	ncfg, err := buildNotifierConfig(&ruler_config.AlertManagerConfig{
+		AlertmanagerURL: "http://alertmanager.default.svc.cluster.local/alertmanager",
+	}, labels.EmptyLabels())
+	require.NoError(t, err)
+	require.Equal(t, model.UTF8Validation, ncfg.GlobalConfig.MetricNameValidationScheme)
+}
+
+// TestNotifierManagerSendWithAlertRelabelReplaceDoesNotPanic exercises the same relabel Replace
+// path as grafana/loki#21368; without UTF8 on synthetic GlobalConfig, ApplyConfig leaves relabel
+// configs unset and Send panics inside ValidationScheme.IsValidLabelName.
+func TestNotifierManagerSendWithAlertRelabelReplaceDoesNotPanic(t *testing.T) {
+	ncfg, err := buildNotifierConfig(&ruler_config.AlertManagerConfig{
+		AlertmanagerURL: "http://127.0.0.1:9093",
+		AlertRelabelConfigs: []*relabel.Config{
+			{
+				Action:       relabel.Replace,
+				SourceLabels: model.LabelNames{"severity"},