[receiver/prometheusreceiver] fix prom receiver config reload (#40780)

#### Description

Fixing a bug where reloading the Prometheus config, caused config
entries of type
[Secret](https://github.com/prometheus/common/blob/95acce133ca2c07a966a71d475fb936fc282db18/config/config.go#L36-L44)
to be replaced with the literal value of `<secret>`.

- Switches to new yaml library which allows for custom marshaling
(shoutout to @ridwanmsharif)
- Adds unit test to ensure reloading promreceiver config doesn't mask
secrets

#### Link to tracking issue

Fixes #40520, #40916

#### Testing

- I've added a unit test, validating the `Reload()` method doesn't mask
secrets for existing `scrape_configs`
- Pointed a local instance at a simple http server to ensure the
credentials are not affected
(#40520 (comment))
- Tested locally it with a `target_allocator` config, which was the
original reason for the `Reload()` method, which introduced the bug
(https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/40103/files#r2097520578)

#### Documentation

N/A

---------

Co-authored-by: George Krajcsovits <[email protected]>

Author: erikburt

.chloggen/fix_prom-receiver-auth.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: receiver/prometheusreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Fixes masking of authentication credentials in Prometheus receiver, when reloading the Prometheus config."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [40520, 40916]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

connector/datadogconnector/go.sum

@@ -238,6 +238,7 @@ require (
 	github.com/go-zookeeper/zk v1.0.4 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
+	github.com/goccy/go-yaml v1.18.0 // indirect
 	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect

exporter/datadogexporter/go.mod

@@ -86,6 +86,7 @@ require (
 	github.com/go-viper/mapstructure/v2 v2.3.0 // indirect
 	github.com/go-zookeeper/zk v1.0.4 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/goccy/go-yaml v1.18.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect

exporter/datadogexporter/go.sum

@@ -11,12 +11,12 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/goccy/go-yaml"
 	commonconfig "github.com/prometheus/common/config"
 	promconfig "github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery/kubernetes"
 	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/confmap"
-	"gopkg.in/yaml.v3"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/targetallocator"
 )
@@ -145,10 +145,16 @@ func (cfg *PromConfig) Validate() error {
 }
 
 func reloadPromConfig(dst *PromConfig, src any) error {
-	yamlOut, err := yaml.Marshal(src)
+	yamlOut, err := yaml.MarshalWithOptions(
+		src,
+		yaml.CustomMarshaler(func(s commonconfig.Secret) ([]byte, error) {
+			return []byte(s), nil
+		}),
+	)
 	if err != nil {
 		return fmt.Errorf("prometheus receiver: failed to marshal config to yaml: %w", err)
 	}
+
 	newCfg, err := promconfig.Load(string(yamlOut), slog.Default())
 	if err != nil {
 		return fmt.Errorf("prometheus receiver: failed to unmarshal yaml to prometheus config object: %w", err)

exporter/datadogexporter/integrationtest/go.sum

@@ -5,13 +5,16 @@ package prometheusreceiver
 
 import (
 	"context"
+	"log/slog"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
 	promConfig "github.com/prometheus/common/config"
 	promModel "github.com/prometheus/common/model"
+	promconfig "github.com/prometheus/prometheus/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component"
@@ -399,3 +402,79 @@ func TestLoadPrometheusAPIServerExtensionConfig(t *testing.T) {
 	require.NoError(t, sub.Unmarshal(cfg))
 	require.Error(t, xconfmap.Validate(cfg))
 }
+
+func TestReloadPromConfigSecretHandling(t *testing.T) {
+	// This test verifies that the Reload() method preserves secrets instead of
+	// corrupting them to "<secret>" placeholders. This is critical for authentication
+	// to work properly when using configurations with basic auth or bearer tokens.
+
+	tests := []struct {
+		name       string
+		configYAML string
+		checkFn    func(t *testing.T, dst *PromConfig)
+	}{
+		{
+			name: "basic auth password preservation",
+			configYAML: `
+scrape_configs:
+  - job_name: "test-basic-auth"
+    basic_auth:
+      username: "testuser"
+      password: "mysecretpassword"
+    static_configs:
+      - targets: ["localhost:8080"]
+`,
+			checkFn: func(t *testing.T, dst *PromConfig) {
+				require.Len(t, dst.ScrapeConfigs, 1)
+				scrapeConfig := dst.ScrapeConfigs[0]
+				assert.Equal(t, "test-basic-auth", scrapeConfig.JobName)
+
+				// The critical check: ensure the password is not "<secret>"
+				require.NotNil(t, scrapeConfig.HTTPClientConfig.BasicAuth, "basic auth should be configured")
+				password := string(scrapeConfig.HTTPClientConfig.BasicAuth.Password)
+				assert.Equal(t, "mysecretpassword", password, "password should preserve original value")
+				assert.Equal(t, "testuser", scrapeConfig.HTTPClientConfig.BasicAuth.Username)
+			},
+		},
+		{
+			name: "bearer token preservation",
+			configYAML: `
+scrape_configs:
+  - job_name: "test-bearer-token"
+    authorization:
+      type: "Bearer"
+      credentials: "mySecretBearerToken123"
+    static_configs:
+      - targets: ["localhost:9090"]
+`,
+			checkFn: func(t *testing.T, dst *PromConfig) {
+				require.Len(t, dst.ScrapeConfigs, 1)
+				scrapeConfig := dst.ScrapeConfigs[0]
+				assert.Equal(t, "test-bearer-token", scrapeConfig.JobName)
+
+				// Check that bearer token is preserved
+				require.NotNil(t, scrapeConfig.HTTPClientConfig.Authorization, "authorization should be configured")
+				credentials := string(scrapeConfig.HTTPClientConfig.Authorization.Credentials)
+				assert.Equal(t, "mySecretBearerToken123", credentials, "credentials should preserve original value")
+				assert.Equal(t, "Bearer", scrapeConfig.HTTPClientConfig.Authorization.Type)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Load the config using promconfig.Load to simulate real usage
+			initialCfg, err := promconfig.Load(tt.configYAML, slog.New(slog.NewTextHandler(os.Stderr, nil)))
+			require.NoError(t, err)
+
+			// Convert to PromConfig and test the Reload method
+			// The Reload method should preserve secrets and not corrupt them
+			dst := (*PromConfig)(initialCfg)
+			err = dst.Reload()
+			require.NoError(t, err)
+
+			// Verify that secrets are preserved
+			tt.checkFn(t, dst)
+		})
+	}
+}

exporter/prometheusexporter/go.mod

@@ -4,6 +4,7 @@ go 1.23.0
 
 require (
 	github.com/go-kit/log v0.2.1
+	github.com/goccy/go-yaml v1.18.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v1.0.0
 	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc
@@ -44,7 +45,6 @@ require (
 	go.uber.org/zap/exp v0.3.0
 	golang.org/x/net v0.41.0
 	google.golang.org/protobuf v1.36.6
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -281,6 +281,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.32.3 // indirect
 	k8s.io/apimachinery v0.32.3 // indirect
 	k8s.io/client-go v0.32.3 // indirect

exporter/prometheusexporter/go.sum

@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/goccy/go-yaml"
 	"github.com/gogo/protobuf/proto"
 	"github.com/prometheus/common/promslog"
 	promcfg "github.com/prometheus/prometheus/config"
@@ -34,7 +35,6 @@ import (
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"go.opentelemetry.io/collector/receiver/receivertest"
 	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
-	"gopkg.in/yaml.v3"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal/metadata"

receiver/prometheusreceiver/config.go

@@ -7,12 +7,12 @@ import (
 	"os"
 	"testing"
 
+	"github.com/goccy/go-yaml"
 	"github.com/prometheus/prometheus/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
-	"gopkg.in/yaml.v3"
 )
 
 var scrapeFileTargetPage = `

receiver/prometheusreceiver/config_test.go

@@ -15,6 +15,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/goccy/go-yaml"
 	promTestUtil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
@@ -28,7 +29,6 @@ import (
 	"go.opentelemetry.io/collector/receiver/receivertest"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
-	"gopkg.in/yaml.v3"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal/metadata"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/targetallocator"

receiver/prometheusreceiver/go.mod

@@ -4,7 +4,6 @@
 package targetallocator // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/targetallocator"
 
 import (
-	"bytes"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -13,11 +12,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/goccy/go-yaml"
 	commonconfig "github.com/prometheus/common/config"
 	promHTTP "github.com/prometheus/prometheus/discovery/http"
 	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/confmap"
-	"gopkg.in/yaml.v3"
 )
 
 type Config struct {
@@ -111,14 +110,17 @@ func checkTLSConfig(tlsConfig commonconfig.TLSConfig) error {
 }
 
 func unmarshalYAML(in map[string]any, out any) error {
-	yamlOut, err := yaml.Marshal(in)
+	yamlOut, err := yaml.MarshalWithOptions(
+		in,
+		yaml.CustomMarshaler[commonconfig.Secret](func(s commonconfig.Secret) ([]byte, error) {
+			return []byte(s), nil
+		}),
+	)
 	if err != nil {
 		return fmt.Errorf("prometheus receiver: failed to marshal config to yaml: %w", err)
 	}
 
-	decoder := yaml.NewDecoder(bytes.NewReader(yamlOut))
-	decoder.KnownFields(true)
-	err = decoder.Decode(out)
+	err = yaml.Unmarshal(yamlOut, out)
 	if err != nil {
 		return fmt.Errorf("prometheus receiver: failed to unmarshal yaml to prometheus config object: %w", err)
 	}

receiver/prometheusreceiver/go.sum

@@ -15,6 +15,7 @@ import (
 	"sort"
 	"time"
 
+	"github.com/goccy/go-yaml"
 	commonconfig "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	promconfig "github.com/prometheus/prometheus/config"
@@ -24,7 +25,6 @@ import (
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/receiver"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v3"
 )
 
 type Manager struct {

receiver/prometheusreceiver/metrics_receiver_helper_test.go

@@ -80,6 +80,7 @@ require (
 	github.com/go-viper/mapstructure/v2 v2.3.0 // indirect
 	github.com/go-zookeeper/zk v1.0.4 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/goccy/go-yaml v1.18.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect

receiver/prometheusreceiver/metrics_receiver_scrape_config_files_test.go

@@ -79,6 +79,7 @@ require (
 	github.com/go-viper/mapstructure/v2 v2.3.0 // indirect
 	github.com/go-zookeeper/zk v1.0.4 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/goccy/go-yaml v1.18.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect

receiver/prometheusreceiver/metrics_receiver_target_allocator_test.go

@@ -76,6 +76,7 @@ require (
 	github.com/go-viper/mapstructure/v2 v2.3.0 // indirect
 	github.com/go-zookeeper/zk v1.0.4 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/goccy/go-yaml v1.18.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect