Otel Agent crash on openshift

[nitinjain999]

2023/10/31 09:15:52 settings.go:436: Set config to [/conf/relay.yaml]
2023/10/31 09:15:52 settings.go:489: Set ballast to 1980 MiB
2023/10/31 09:15:52 settings.go:505: Set memory limit to 5400 MiB
2023/10/31 09:15:52 settings.go:396: set "SPLUNK_LISTEN_INTERFACE" to "0.0.0.0"
2023-10-31T09:15:52.760Z info [email protected]/telemetry.go:84 Setting up own telemetry...
2023-10-31T09:15:52.760Z info [email protected]/telemetry.go:201 Serving Prometheus metrics {"address": "0.0.0.0:8889", "level": "Basic"}
2023-10-31T09:15:52.763Z info kube/client.go:107 k8s filtering {"kind": "processor", "name": "k8sattributes", "pipeline": "logs", "labelSelector": "", "fieldSelector": "spec.nodeName=worker06.devtest.openshift.hh.atg.se"}
2023-10-31T09:15:52.763Z info [email protected]/memorylimiter.go:102 Memory limiter configured {"kind": "processor", "name": "memory_limiter", "pipeline": "logs", "limit_mib": 5400, "spike_limit_mib": 1080, "check_interval": 2}
2023-10-31T09:15:52.767Z info [email protected]/service.go:138 Starting otelcol... {"Version": "v0.86.0", "NumCPU": 12}
2023-10-31T09:15:52.767Z info extensions/extensions.go:31 Starting extensions...
2023-10-31T09:15:52.767Z info extensions/extensions.go:34 Extension is starting... {"kind": "extension", "name": "file_storage"}
2023-10-31T09:15:52.767Z info extensions/extensions.go:38 Extension started. {"kind": "extension", "name": "file_storage"}
2023-10-31T09:15:52.767Z info extensions/extensions.go:34 Extension is starting... {"kind": "extension", "name": "health_check"}
2023-10-31T09:15:52.767Z info [email protected]/healthcheckextension.go:35 Starting health_check extension {"kind": "extension", "name": "health_check", "config": {"Endpoint":"0.0.0.0:13133","TLSSetting":null,"CORS":null,"Auth":null,"MaxRequestBodySize":0,"IncludeMetadata":false,"ResponseHeaders":null,"Path":"/","ResponseBody":null,"CheckCollectorPipeline":{"Enabled":false,"Interval":"5m","ExporterFailureThreshold":5}}}
2023-10-31T09:15:52.768Z warn [email protected]/warning.go:40 Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks {"kind": "extension", "name": "health_check", "documentation": "https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks"}
2023-10-31T09:15:52.768Z info extensions/extensions.go:38 Extension started. {"kind": "extension", "name": "health_check"}
2023-10-31T09:15:52.768Z info extensions/extensions.go:34 Extension is starting... {"kind": "extension", "name": "k8s_observer"}
2023-10-31T09:15:52.768Z info extensions/extensions.go:38 Extension started. {"kind": "extension", "name": "k8s_observer"}
2023-10-31T09:15:52.768Z info extensions/extensions.go:34 Extension is starting... {"kind": "extension", "name": "memory_ballast"}
2023-10-31T09:15:53.400Z info [email protected]/memory_ballast.go:41 Setting memory ballast {"kind": "extension", "name": "memory_ballast", "MiBs": 1980}
2023-10-31T09:15:53.400Z info extensions/extensions.go:38 Extension started. {"kind": "extension", "name": "memory_ballast"}
2023-10-31T09:15:53.400Z info extensions/extensions.go:34 Extension is starting... {"kind": "extension", "name": "zpages"}
2023-10-31T09:15:53.400Z info [email protected]/zpagesextension.go:53 Registered zPages span processor on tracer provider {"kind": "extension", "name": "zpages"}
2023-10-31T09:15:53.400Z info [email protected]/zpagesextension.go:63 Registered Host's zPages {"kind": "extension", "name": "zpages"}
2023-10-31T09:15:53.400Z info [email protected]/zpagesextension.go:75 Starting zPages extension {"kind": "extension", "name": "zpages", "config": {"TCPAddr":{"Endpoint":"localhost:55679"}}}
2023-10-31T09:15:53.401Z info extensions/extensions.go:38 Extension started. {"kind": "extension", "name": "zpages"}
2023-10-31T09:15:53.401Z info internal/resourcedetection.go:125 began detecting resource information {"kind": "processor", "name": "resourcedetection", "pipeline": "logs"}
2023-10-31T09:15:53.403Z info internal/resourcedetection.go:139 detected resource information {"kind": "processor", "name": "resourcedetection", "pipeline": "logs", "resource": {"host.name":"worker06.devtest.openshift.hh.atg.se","os.type":"linux"}}
2023-10-31T09:15:53.403Z warn [email protected]/warning.go:40 Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks {"kind": "receiver", "name": "otlp", "data_type": "logs", "documentation": "https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks"}
2023-10-31T09:15:53.403Z info [email protected]/otlp.go:83 Starting GRPC server {"kind": "receiver", "name": "otlp", "data_type": "logs", "endpoint": "0.0.0.0:4317"}
2023-10-31T09:15:53.404Z warn [email protected]/warning.go:40 Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks {"kind": "receiver", "name": "otlp", "data_type": "logs", "documentation": "https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks"}
2023-10-31T09:15:53.404Z info [email protected]/otlp.go:101 Starting HTTP server {"kind": "receiver", "name": "otlp", "data_type": "logs", "endpoint": "0.0.0.0:4318"}
2023-10-31T09:15:53.404Z info adapter/receiver.go:45 Starting stanza receiver {"kind": "receiver", "name": "filelog", "data_type": "logs"}
panic: assertion failed: Page expected to be: 1491, but self identifies as 7659861336726075989

goroutine 1 [running]:
go.etcd.io/bbolt._assert(...)
go.etcd.io/[email protected]/db.go:1359
go.etcd.io/bbolt.(*page).fastCheck(0x7fe0317d3000, 0x5d3)
go.etcd.io/[email protected]/page.go:57 +0x1df
go.etcd.io/bbolt.(*Tx).page(0x7fe05b4fb108?, 0x40?)
go.etcd.io/[email protected]/tx.go:534 +0x8a
go.etcd.io/bbolt.(*Tx).forEachPageInternal(0xc0007f2f00?, {0xc001e52410?, 0x1, 0xa}, 0xc07ec8e520)
go.etcd.io/[email protected]/tx.go:546 +0x65
go.etcd.io/bbolt.(*Tx).forEachPage(...)
go.etcd.io/[email protected]/tx.go:542
go.etcd.io/bbolt.(*Tx).checkBucket(0xc002d02b60, 0xc0003bd2c0, 0xc07ec8e8c0, 0xc07ec8e890, {0x6b55f78?, 0xa2c3b58}, 0xc0007f2f00)
go.etcd.io/[email protected]/tx_check.go:83 +0x126
go.etcd.io/bbolt.(*Tx).checkBucket.func2({0x7fe031202020?, 0xc07ec8e6e0?, 0xc002d02b60?})
go.etcd.io/[email protected]/tx_check.go:110 +0x93
go.etcd.io/bbolt.(*Bucket).ForEachBucket(0xc002d02b78, 0xc07ec8e728)
go.etcd.io/[email protected]/bucket.go:403 +0xb0
go.etcd.io/bbolt.(*Tx).checkBucket(0xc002d02b60, 0xc002d02b78, 0xc07ec8e8c0, 0xc07ec8e890, {0x6b55f78?, 0xa2c3b58}, 0xc0007f2f00)
go.etcd.io/[email protected]/tx_check.go:108 +0x265
go.etcd.io/bbolt.(*DB).freepages(0x5fec61a?)
go.etcd.io/[email protected]/db.go:1181 +0x229
go.etcd.io/bbolt.(*DB).loadFreelist.func1()
go.etcd.io/[email protected]/db.go:412 +0xc5
sync.(*Once).doSlow(0xc0021161c8?, 0x10?)
sync/once.go:74 +0xc2
sync.(*Once).Do(...)
sync/once.go:65
go.etcd.io/bbolt.(*DB).loadFreelist(0xc002116000?)
go.etcd.io/[email protected]/db.go:408 +0x47
go.etcd.io/bbolt.Open({0xc0000787b0, 0x2c}, 0x1?, 0xc07ec8eb68)
go.etcd.io/[email protected]/db.go:290 +0x40c
github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage.newClient(0xc002c4e930, {0xc0000787b0?, 0xc07ec8ec50?}, 0x3b9aca00, 0xc002bced00)
github.com/open-telemetry/opentelemetry-collector-contrib/extension/[email protected]/filestorage/client.go:51 +0xaf
github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage.(*localFileStorage).GetClient(0xc002c5bf50, {0xc1484f5258163834?, 0x0?}, 0xc1484f5258163834?, {{0xc002bf9238?, 0xa282c20?}, {0x0?, 0x0?}}, {0x0, 0x0})
github.com/open-telemetry/opentelemetry-collector-contrib/extension/[email protected]/filestorage/extension.go:54 +0x355
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/adapter.GetStorageClient({0x6b781d8, 0xc00013a000}, {0x6b7b038?, 0xc002c4e380?}, 0xc002bb34c0, {{0xc002bf9238?, 0x0?}, {0x0?, 0x0?}})
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/adapter/storage.go:29 +0x108
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/adapter.(*receiver).setStorageClient(0xc002d54ea0, {0x6b781d8?, 0xc00013a000?}, {0x6b7b038?, 0xc002c4e380?})
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/adapter/storage.go:34 +0x58
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/adapter.(*receiver).Start(0xc002d54ea0, {0x6b781d8, 0xc00013a000}, {0x6b7b038, 0xc002c4e380})
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/adapter/receiver.go:47 +0xba
go.opentelemetry.io/collector/service/internal/graph.(*Graph).StartAll(0xc002bcfdc0?, {0x6b781d8, 0xc00013a000}, {0x6b7b038, 0xc002c4e380})
go.opentelemetry.io/collector/[email protected]/internal/graph/graph.go:334 +0xb3
go.opentelemetry.io/collector/service.(*Service).Start(0xc00086ec60, {0x6b781d8, 0xc00013a000})
go.opentelemetry.io/collector/[email protected]/service.go:153 +0x2e5
go.opentelemetry.io/collector/otelcol.(*Collector).setupConfigurationComponents(0xc001ea8f00, {0x6b781d8, 0xc00013a000})
go.opentelemetry.io/collector/[email protected]/collector.go:186 +0x6cb
go.opentelemetry.io/collector/otelcol.(*Collector).Run(0xc001ea8f00, {0x6b781d8, 0xc00013a000})
go.opentelemetry.io/collector/[email protected]/collector.go:221 +0x65
go.opentelemetry.io/collector/otelcol.NewCommand.func1(0xc00236ac00, {0x5fed7ee?, 0x0?, 0x1?})
go.opentelemetry.io/collector/[email protected]/command.go:27 +0x96
github.com/spf13/cobra.(*Command).execute(0xc00236ac00, {0xc000134010, 0x0, 0x1})
github.com/spf13/[email protected]/command.go:940 +0x862
github.com/spf13/cobra.(*Command).ExecuteC(0xc00236ac00)
github.com/spf13/[email protected]/command.go:1068 +0x3bd
github.com/spf13/cobra.(*Command).Execute(0xc002379290?)
github.com/spf13/[email protected]/command.go:992 +0x19
main.runInteractive({{0xc002379290, 0xc002379500, 0xc002379440, 0xc002379020, 0xc002379530}, {{0x5fecd91, 0x7}, {0x0, 0x0}, {0x6b06c98, ...}}, ...})
github.com/signalfx/splunk-otel-collector/cmd/otelcol/main.go:100 +0x5d
main.run(...)
github.com/signalfx/splunk-otel-collector/cmd/otelcol/main_others.go:26
main.main()
github.com/signalfx/splunk-otel-collector/cmd/otelcol/main.go:93 +0xcba