[extension/memorylimiter] add memory limiter extension, a copy of memorylimiter processor

.chloggen/extension-memory-limiter.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: extension/memory_limiter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Introduce a `memory_limiter` extension which receivers can use to reject incoming requests when collector doesn't have enough memory
+
+# One or more tracking issues or pull requests related to the change
+issues: [8632]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: The extension has the same configuration interface and behavior as the existing `memory_limiter` processor, which potentially can be deprecated and removed in the future
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

cmd/otelcorecol/builder-config.yaml

@@ -14,6 +14,7 @@ exporters:
   - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.92.0
 extensions:
   - gomod: go.opentelemetry.io/collector/extension/ballastextension v0.92.0
+  - gomod: go.opentelemetry.io/collector/extension/memorylimiterextension v0.92.0
   - gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.92.0
 processors:
   - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.92.0
@@ -47,6 +48,7 @@ replaces:
   - go.opentelemetry.io/collector/extension => ../../extension
   - go.opentelemetry.io/collector/extension/auth => ../../extension/auth
   - go.opentelemetry.io/collector/extension/ballastextension => ../../extension/ballastextension
+  - go.opentelemetry.io/collector/extension/memorylimiterextension => ../../extension/memorylimiterextension
   - go.opentelemetry.io/collector/extension/zpagesextension => ../../extension/zpagesextension
   - go.opentelemetry.io/collector/featuregate => ../../featuregate
   - go.opentelemetry.io/collector/pdata => ../../pdata

cmd/otelcorecol/go.mod

@@ -16,6 +16,7 @@ require (
 	go.opentelemetry.io/collector/exporter/otlphttpexporter v0.92.0
 	go.opentelemetry.io/collector/extension v0.92.0
 	go.opentelemetry.io/collector/extension/ballastextension v0.92.0
+	go.opentelemetry.io/collector/extension/memorylimiterextension v0.92.0
 	go.opentelemetry.io/collector/extension/zpagesextension v0.92.0
 	go.opentelemetry.io/collector/otelcol v0.92.0
 	go.opentelemetry.io/collector/processor v0.92.0
@@ -178,6 +179,8 @@ replace go.opentelemetry.io/collector/extension/auth => ../../extension/auth
 
 replace go.opentelemetry.io/collector/extension/ballastextension => ../../extension/ballastextension
 
+replace go.opentelemetry.io/collector/extension/memorylimiterextension => ../../extension/memorylimiterextension
+
 replace go.opentelemetry.io/collector/extension/zpagesextension => ../../extension/zpagesextension
 
 replace go.opentelemetry.io/collector/featuregate => ../../featuregate

extension/memorylimiterextension/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

extension/memorylimiterextension/README.md

@@ -0,0 +1,22 @@
+# Memory Limiter Extension
+
+> [!WARNING]
+> The memory_limiter extension cannot be used if the deprecated memory_ballast extension is enabled.
+
+<!-- status autogenerated section -->
+| Status        |           |
+| ------------- |-----------|
+| Stability     | [development]  |
+| Distributions | [] |
+| Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aextension%2Fmemorylimiter%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aextension%2Fmemorylimiter) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aextension%2Fmemorylimiter%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aextension%2Fmemorylimiter) |
+
+[development]: https://github.com/open-telemetry/opentelemetry-collector#development
+<!-- end autogenerated section -->
+
+The memory limiter extension is used to prevent out of memory situations on
+the collector. The extension will potentially replace the Memory Limiter Processor. 
+It provides better guarantees from running out of memory as it will be used by the 
+receivers to reject requests before converting them into OTLP. All the configurations 
+are the same as Memory Limiter Processor. The extension is under development and does nothing.
+
+see [memorylimiterprocessor](../../processor/memorylimiterprocessor/README.md) for additional details

extension/memorylimiterextension/config.go

@@ -0,0 +1,10 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package memorylimiterextension // import "go.opentelemetry.io/collector/extension/memorylimiterextension"
+
+import (
+	"go.opentelemetry.io/collector/internal/memorylimiter"
+)
+
+type Config = memorylimiter.Config

extension/memorylimiterextension/factory.go

@@ -0,0 +1,33 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package memorylimiterextension // import "go.opentelemetry.io/collector/extension/memorylimiterextension"
+
+//go:generate mdatagen metadata.yaml
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/extension"
+	"go.opentelemetry.io/collector/extension/memorylimiterextension/internal/metadata"
+)
+
+// NewFactory returns a new factory for the Memory Limiter extension.
+func NewFactory() extension.Factory {
+	return extension.NewFactory(
+		metadata.Type,
+		createDefaultConfig,
+		createExtension,
+		metadata.ExtensionStability)
+}
+
+// CreateDefaultConfig creates the default configuration for extension. Notice
+// that the default configuration is expected to fail for this extension.
+func createDefaultConfig() component.Config {
+	return &Config{}
+}
+
+func createExtension(_ context.Context, set extension.CreateSettings, cfg component.Config) (extension.Extension, error) {
+	return newMemoryLimiter(cfg.(*Config), set.TelemetrySettings.Logger)
+}

extension/memorylimiterextension/factory_test.go

@@ -0,0 +1,50 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package memorylimiterextension
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/extension/extensiontest"
+	"go.opentelemetry.io/collector/internal/memorylimiter"
+)
+
+func TestCreateDefaultConfig(t *testing.T) {
+	factory := NewFactory()
+	require.NotNil(t, factory)
+
+	cfg := factory.CreateDefaultConfig()
+	assert.NotNil(t, cfg, "failed to create default config")
+	assert.NoError(t, componenttest.CheckConfigStruct(cfg))
+}
+
+func TestCreateExtension(t *testing.T) {
+	factory := NewFactory()
+	require.NotNil(t, factory)
+
+	cfg := factory.CreateDefaultConfig()
+
+	// Create extension with a valid config.
+	pCfg := cfg.(*Config)
+	pCfg.MemoryLimitMiB = 5722
+	pCfg.MemorySpikeLimitMiB = 1907
+	pCfg.CheckInterval = 100 * time.Millisecond
+
+	tp, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg)
+	assert.NoError(t, err)
+	assert.NotNil(t, tp)
+	// test if we can shutdown a monitoring routine that has not started
+	assert.ErrorIs(t, tp.Shutdown(context.Background()), memorylimiter.ErrShutdownNotStarted)
+	assert.NoError(t, tp.Start(context.Background(), componenttest.NewNopHost()))
+
+	assert.NoError(t, tp.Shutdown(context.Background()))
+	// verify that no monitoring routine is running
+	assert.ErrorIs(t, tp.Shutdown(context.Background()), memorylimiter.ErrShutdownNotStarted)
+}

extension/memorylimiterextension/go.mod

@@ -0,0 +1,81 @@
+module go.opentelemetry.io/collector/extension/memorylimiterextension
+
+go 1.20
+
+require (
+	github.com/stretchr/testify v1.8.4
+	go.opentelemetry.io/collector v0.92.0
+	go.opentelemetry.io/collector/component v0.92.0
+	go.opentelemetry.io/collector/extension v0.92.0
+	go.opentelemetry.io/otel/metric v1.22.0
+	go.opentelemetry.io/otel/trace v1.22.0
+	go.uber.org/zap v1.26.0
+)
+
+require (
+	contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-logfmt/logfmt v0.5.1 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
+	github.com/knadh/koanf/v2 v2.0.1 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.46.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/prometheus/statsd_exporter v0.22.7 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/collector/config/configtelemetry v0.92.0 // indirect
+	go.opentelemetry.io/collector/confmap v0.92.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.0.1 // indirect
+	go.opentelemetry.io/collector/pdata v1.0.1 // indirect
+	go.opentelemetry.io/otel v1.22.0 // indirect
+	go.opentelemetry.io/otel/exporters/prometheus v0.44.1-0.20231201153405-6027c1ae76f2 // indirect
+	go.opentelemetry.io/otel/sdk v1.22.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.22.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
+	google.golang.org/grpc v1.60.1 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace go.opentelemetry.io/collector => ../../
+
+replace go.opentelemetry.io/collector/component => ../../component
+
+replace go.opentelemetry.io/collector/confmap => ../../confmap
+
+replace go.opentelemetry.io/collector/extension => ../../extension
+
+replace go.opentelemetry.io/collector/featuregate => ../../featuregate
+
+replace go.opentelemetry.io/collector/pdata => ../../pdata
+
+replace go.opentelemetry.io/collector/consumer => ../../consumer
+
+replace go.opentelemetry.io/collector/config/configtelemetry => ../../config/configtelemetry