diff --git a/.chloggen/fix_red-metrics.yaml b/.chloggen/fix_red-metrics.yaml index c778c76e8..993f0a1b7 100755 --- a/.chloggen/fix_red-metrics.yaml +++ b/.chloggen/fix_red-metrics.yaml @@ -17,6 +17,22 @@ issues: [1072] subtext: | Fix RED metric retrieval from Prometheus by removing the default RED metric namespace introduced in Jaeger 1.62. The Operator overwrites the new default namespace jaeger-query takes into account using the `--prometheus.query.namespace=` flag. + A user can overwrite the default namespace by configuring it in the CR. + Example: + ``` + apiVersion: tempo.grafana.com/v1alpha1 + kind: TempoStack + ... + spec: + template: + queryFrontend: + jaegerQuery: + enabled: true + monitorTab: + enabled: true + prometheusEndpoint: "http://myPromInstance:9090" + redMetricsNamespace: "custom" + ``` More details can be found here: - https://github.com/jaegertracing/jaeger/pull/6007 - https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/34485 diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index f71123172..eac411d9d 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -689,6 +689,12 @@ type JaegerQueryMonitor struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Prometheus endpoint" PrometheusEndpoint string `json:"prometheusEndpoint"` + // REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. + // By default no namespace is set. + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="RED Metric Namespace" + REDMetricsNamespace string `json:"redMetricsNamespace"` } // IngressSpec defines Jaeger Query Ingress options. diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index 103fd3d01..9c4f62f2a 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -1070,6 +1070,10 @@ spec: on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint + - description: REDMetricsNamespace defines the a prefix used retrieve span rate, + error, and duration (RED) metrics. By default no namespace is set. + displayName: RED Metric Namespace + path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override the calculated resources derived from total displayName: Resources diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index ad90ab922..27cf84b96 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -2470,6 +2470,11 @@ spec: PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 type: string + redMetricsNamespace: + description: |- + REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. + By default no namespace is set. + type: string type: object resources: description: Resources defines resources for this component, diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 7a2ead8e5..230cc34cc 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -1070,6 +1070,10 @@ spec: on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint + - description: REDMetricsNamespace defines the a prefix used retrieve span rate, + error, and duration (RED) metrics. By default no namespace is set. + displayName: RED Metric Namespace + path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override the calculated resources derived from total displayName: Resources diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index ad90ab922..27cf84b96 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -2470,6 +2470,11 @@ spec: PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 type: string + redMetricsNamespace: + description: |- + REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. + By default no namespace is set. + type: string type: object resources: description: Resources defines resources for this component, diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index da0c38c5e..3d36981d9 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -2466,6 +2466,11 @@ spec: PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 type: string + redMetricsNamespace: + description: |- + REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. + By default no namespace is set. + type: string type: object resources: description: Resources defines resources for this component, diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml index 6500bcdf8..95f3991f9 100644 --- a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml @@ -999,6 +999,10 @@ spec: on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint + - description: REDMetricsNamespace defines the a prefix used retrieve span rate, + error, and duration (RED) metrics. By default no namespace is set. + displayName: RED Metric Namespace + path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override the calculated resources derived from total displayName: Resources diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml index 4f3f7f264..5f7328a6e 100644 --- a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml @@ -999,6 +999,10 @@ spec: on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint + - description: REDMetricsNamespace defines the a prefix used retrieve span rate, + error, and duration (RED) metrics. By default no namespace is set. + displayName: RED Metric Namespace + path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override the calculated resources derived from total displayName: Resources diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index a7641ae03..2153b3629 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -342,6 +342,7 @@ spec: # TempoStackSpec defines the desired st monitorTab: # MonitorTab defines the monitor tab configuration. enabled: false # Enabled enables the monitor tab in the Jaeger console. The PrometheusEndpoint must be configured to enable this feature. prometheusEndpoint: "" # PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 + redMetricsNamespace: "" # REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. By default no namespace is set. servicesQueryDuration: "" # ServicesQueryDuration defines how long the services will be available in the services list tempoQuery: # TempoQuery defines options specific to the Tempoo Query component. resources: # Resources defines resources for this component, this will override the calculated resources derived from total diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go index 49b125b44..55d1afe70 100644 --- a/internal/manifests/queryfrontend/query_frontend.go +++ b/internal/manifests/queryfrontend/query_frontend.go @@ -402,7 +402,7 @@ func enableMonitoringTab(tempo v1alpha1.TempoStack, jaegerQueryContainer corev1. // NOTE: Jaeger 1.62 default namespace changed to "traces_span_metrics". // We fallback to no namespace. // See https://github.com/jaegertracing/jaeger/pull/6007. - "--prometheus.query.namespace=", + fmt.Sprintf("--prometheus.query.namespace=%s", tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace), }, } // If the endpoint matches Prometheus on OpenShift, configure TLS and token based auth diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go index 778303adf..9ffb26b6a 100644 --- a/internal/manifests/queryfrontend/query_frontend_test.go +++ b/internal/manifests/queryfrontend/query_frontend_test.go @@ -621,15 +621,16 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) { JaegerQuery: v1alpha1.JaegerQuerySpec{ Enabled: true, MonitorTab: v1alpha1.JaegerQueryMonitor{ - Enabled: true, - PrometheusEndpoint: "http://prometheus:9091", + Enabled: true, + PrometheusEndpoint: "http://prometheus:9091", + REDMetricsNamespace: "test", }, }, }, }, }, }, - args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace="}, + args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace=test"}, env: []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}}, }, {