Otelhttp does not work with Prometheus 3.0

[Nessaek]

Description

I'm using otelhttp to generate and create metrics that are sent to Prometheus. However it only works with versions of Prometheus lower than 3.0. If you use Prometheus 3.0 and above, the metrics don't get converted into Prometheus friendly ones. E.g with prometheus 3.0 the metric will look like http.server.request.body.size.bytes.bucket . With a lower version it will be converted to something prometheus can read e.g. http_server_request_body_size_bytes_bucket.

Is this a known issue?

Environment

• OS:Mac
• Architecture: arm64
• Go Version: 1.24
• otelhttp version: v0.61.0

Steps To Reproduce

1. Using this code ...

My main file:

package main

import (
	"context"
	//"context"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"

	"github.com/prometheus/client_golang/prometheus/promhttp"
	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"

	"rfc/handlers"
	"rfc/observability"

	//"path/filepath"
	"rfc/service"
)

func main() {

	collectorEndpoint := os.Getenv("COLLECTOR_ENDPOINT")

	err := observability.InitObservability(context.Background(), collectorEndpoint)

	if err != nil {
		log.Fatalf("Failed to initialize tracer: %s", err)
	}
	port := "8443"

	cache := service.NewCache()
	handler := handlers.NewHandler(cache)
	instrumentedHandler := otelhttp.NewHandler(otelhttp.WithRouteTag("/validate", serve(handler.ValidationHandler())), "validate")

	http.Handle("/validate", instrumentedHandler)

	http.Handle("/metrics", promhttp.Handler())

	log.Printf("Listening on port %s", port)

	if err := http.Listen(fmt.Sprintf(":%s", port), nil); err != nil {
		log.Fatalf("Failed to start server: %s", err)
	}

}

my observability file:

package observability

import (
	"context"
	"os"

	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
	"go.opentelemetry.io/otel/exporters/prometheus"
	"go.opentelemetry.io/otel/propagation"
	"go.opentelemetry.io/otel/sdk/metric"
	sdkresource "go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
	"go.opentelemetry.io/otel/trace"
	"go.opentelemetry.io/otel/trace/noop"
)

var (
	Tracer                              = otel.Tracer("Tracer")
	Meter                               = otel.Meter("Meter")
	tracerProvider trace.TracerProvider = noop.NewTracerProvider()
)

func InitObservability(ctx context.Context, endpoint string) error {
	exporter, err := newExporter(ctx, endpoint)

	res := buildResource()
	if err != nil {
		return err
	}
	tracerProvider, err = newTraceProvider(exporter, res)
	if err != nil {
		return err
	}
	otel.SetTracerProvider(tracerProvider)

	otel.SetTextMapPropagator(tracePropagator())

	meterProvider, err := newMeterProvider()
	if err != nil {
		return err
	}

	otel.SetMeterProvider(meterProvider)

	return nil
}

func newExporter(ctx context.Context, endpoint string) (sdktrace.SpanExporter, error) {
	options := []otlptracehttp.Option{otlptracehttp.WithEndpoint(endpoint), otlptracehttp.WithInsecure()}

	return otlptracehttp.New(ctx, options...)
}

func newTraceProvider(exp sdktrace.SpanExporter, res *sdkresource.Resource) (*sdktrace.TracerProvider, error) {

	return sdktrace.NewTracerProvider(
		sdktrace.WithBatcher(exp),
		sdktrace.WithResource(res),
	), nil
}

func tracePropagator() propagation.TextMapPropagator {
	return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
}

func buildResource() *sdkresource.Resource {
	hostname, _ := os.Hostname()
	resource := sdkresource.NewWithAttributes(
		semconv.SchemaURL,
		semconv.ServiceName("controller"),
		semconv.ServiceInstanceID(hostname),
		semconv.DeploymentEnvironment("localhost"),
	)

	return resource
}

func newMeterProvider() (*metric.MeterProvider, error) {
	metricExporter, err := prometheus.New()
	if err != nil {
		return nil, err
	}

	meterProvider := metric.NewMeterProvider(
		metric.WithReader(metricExporter),
	)
	return meterProvider, nil
}

3. Run ...
4. Run: go run main.go
5. Run a local prometheus and otel collector using docker compose e.g.

services:
  webhook:
    container_name: webhook
    build:
      context: .
      dockerfile: Dockerfile
    environment:
      COLLECTOR_ENDPOINT: jaeger:4318
    entrypoint: [ "/webhook" ]
    ports:
      - 8080:8080

  jaeger:
    image: jaegertracing/all-in-one:latest
    ports:
      - "16686:16686"
      - "4317:4317"
      - "4318:4318"
    environment:
      - LOG_LEVEL=debug

  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    volumes:
      - ./prometheus-config.yml:/etc/prometheus/prometheus.yml
    ports:
      - 9090:9090

5. See error ...### Expected behavior
   Look at the prometheus metrics - the otelhttp metrics won't be converted properly - they will still have periods (.) in them

[dmathieu]

This is not an otelhttp thing. It's a prometheus exporter.
What versions of the OTel API and SDK are you using?

See the first changelog entry in the Changed section here.
https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md#changed-1

You probably want to use the LegacyValidation scheme (and plan a migration to the prometheus UTF8Validation scheme).

[Nessaek]

This is not an otelhttp thing. It's a prometheus exporter. What versions of the OTel API and SDK are you using?

See the first changelog entry in the Changed section here. https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md#changed-1

You probably want to use the LegacyValidation scheme (and plan a migration to the prometheus UTF8Validation scheme).

thanks - I'm using the latest version of the otel sdk - v1.36.0 so it should already work no? I'm running this on kubernetes with the kube-prometheus-stack helm chart if that makes a difference.

[dmathieu]

You need to use the legacy validation scheme, as mentioned above.
You need to plan your migration to the new validation scheme though, as prometheus may not support the legacy one forever.

[MrAlias]

Closing based on @dmathieu's last comment.