From 41b3f41c0a0dee0d54f9a67fe26551d43fd92f03 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 21 Jul 2025 16:59:10 +0000 Subject: [PATCH 01/17] Add DuplicateFilter to http/grpc exporter --- .../otlp/proto/common/_internal/__init__.py | 10 ++++++++++ .../tests/test_common.py | 17 +++++++++++++++++ .../exporter/otlp/proto/grpc/exporter.py | 5 +++++ .../otlp/proto/http/_log_exporter/__init__.py | 5 +++++ 4 files changed, 37 insertions(+) create mode 100644 exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py index 200644368d..284da1e6a8 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -26,6 +26,7 @@ Optional, TypeVar, ) +import time from opentelemetry.proto.common.v1.common_pb2 import AnyValue as PB2AnyValue from opentelemetry.proto.common.v1.common_pb2 import ( @@ -51,6 +52,15 @@ _ResourceDataT = TypeVar("_ResourceDataT") +class DuplicateFilter(logging.Filter): + def filter(self, record): + current_log = (record.module, record.levelno, record.msg, time.time() // 60) + if current_log != getattr(self, "last_log", None): + self.last_log = current_log + return True + return False + + def _encode_instrumentation_scope( instrumentation_scope: InstrumentationScope, ) -> PB2InstrumentationScope: diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py new file mode 100644 index 0000000000..33cee019a4 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py @@ -0,0 +1,17 @@ +import unittest +import logging + +from opentelemetry.exporter.otlp.proto.common._internal import ( + DuplicateFilter, +) + + +class TestCommonFuncs(unittest.TestCase): + + def test_duplicate_logs_filter_Works(self): + test_logger = logging.getLogger("testLogger") + test_logger.addFilter(DuplicateFilter()) + with self.assertLogs('testLogger') as cm: + test_logger.info("message") + test_logger.info("message") + self.assertEqual(len(cm.output), 1) \ No newline at end of file diff --git a/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py b/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py index c41f7219ae..a6a2efec7b 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py @@ -32,6 +32,9 @@ TypeVar, Union, ) +from opentelemetry.exporter.otlp.proto.common._internal import ( + DuplicateFilter, +) from typing import Sequence as TypingSequence from urllib.parse import urlparse @@ -87,6 +90,8 @@ ) _MAX_RETRYS = 6 logger = getLogger(__name__) +# This prevents logs generated when a log fails to be written to generate another log which fails to be written etc. etc. +logger.addFilter(DuplicateFilter()) SDKDataT = TypeVar("SDKDataT") ResourceDataT = TypeVar("ResourceDataT") TypingResourceT = TypeVar("TypingResourceT") diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py index c64f269b9e..71af2fd585 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py @@ -53,9 +53,14 @@ OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, OTEL_EXPORTER_OTLP_TIMEOUT, ) +from opentelemetry.exporter.otlp.proto.common._internal import ( + DuplicateFilter, +) from opentelemetry.util.re import parse_env_headers _logger = logging.getLogger(__name__) +# This prevents logs generated when a log fails to be written to generate another log which fails to be written etc. etc. +_logger.addFilter(DuplicateFilter()) DEFAULT_COMPRESSION = Compression.NoCompression From 75aacdeb918ac44376a302f90e849a581af60388 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 21 Jul 2025 17:03:14 +0000 Subject: [PATCH 02/17] Precommit and Changelog --- CHANGELOG.md | 2 ++ .../exporter/otlp/proto/common/_internal/__init__.py | 9 +++++++-- .../tests/test_common.py | 7 +++---- .../opentelemetry/exporter/otlp/proto/grpc/exporter.py | 4 +--- .../exporter/otlp/proto/http/_log_exporter/__init__.py | 6 +++--- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81e52e6d40..c74d05736f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Overwrite logging.config.fileConfig and logging.config.dictConfig to ensure the OTLP `LogHandler` remains attached to the root logger. Fix a bug that can cause a deadlock to occur over `logging._lock` in some cases ([#4636](https://github.com/open-telemetry/opentelemetry-python/pull/4636)). +- Filter duplicate logs emitted from the OTLP exporters to avoid endlessly logging when the OTLP logger itself +is failing to export logs. ## Version 1.35.0/0.56b0 (2025-07-11) diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py index 284da1e6a8..a3fd615680 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -16,6 +16,7 @@ from __future__ import annotations import logging +import time from collections.abc import Sequence from typing import ( Any, @@ -26,7 +27,6 @@ Optional, TypeVar, ) -import time from opentelemetry.proto.common.v1.common_pb2 import AnyValue as PB2AnyValue from opentelemetry.proto.common.v1.common_pb2 import ( @@ -54,7 +54,12 @@ class DuplicateFilter(logging.Filter): def filter(self, record): - current_log = (record.module, record.levelno, record.msg, time.time() // 60) + current_log = ( + record.module, + record.levelno, + record.msg, + time.time() // 60, + ) if current_log != getattr(self, "last_log", None): self.last_log = current_log return True diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py index 33cee019a4..fbf0276158 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py @@ -1,5 +1,5 @@ -import unittest import logging +import unittest from opentelemetry.exporter.otlp.proto.common._internal import ( DuplicateFilter, @@ -7,11 +7,10 @@ class TestCommonFuncs(unittest.TestCase): - def test_duplicate_logs_filter_Works(self): test_logger = logging.getLogger("testLogger") test_logger.addFilter(DuplicateFilter()) - with self.assertLogs('testLogger') as cm: + with self.assertLogs("testLogger") as cm: test_logger.info("message") test_logger.info("message") - self.assertEqual(len(cm.output), 1) \ No newline at end of file + self.assertEqual(len(cm.output), 1) diff --git a/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py b/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py index a6a2efec7b..fb92d4e0fe 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py @@ -32,9 +32,6 @@ TypeVar, Union, ) -from opentelemetry.exporter.otlp.proto.common._internal import ( - DuplicateFilter, -) from typing import Sequence as TypingSequence from urllib.parse import urlparse @@ -51,6 +48,7 @@ ssl_channel_credentials, ) from opentelemetry.exporter.otlp.proto.common._internal import ( + DuplicateFilter, _get_resource_data, ) from opentelemetry.exporter.otlp.proto.grpc import ( diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py index 71af2fd585..5de62c9f91 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py @@ -24,6 +24,9 @@ import requests from requests.exceptions import ConnectionError +from opentelemetry.exporter.otlp.proto.common._internal import ( + DuplicateFilter, +) from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs from opentelemetry.exporter.otlp.proto.http import ( _OTLP_HTTP_HEADERS, @@ -53,9 +56,6 @@ OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, OTEL_EXPORTER_OTLP_TIMEOUT, ) -from opentelemetry.exporter.otlp.proto.common._internal import ( - DuplicateFilter, -) from opentelemetry.util.re import parse_env_headers _logger = logging.getLogger(__name__) From 0c1aadf1d66d07230b276aea5da96252d108e79e Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 21 Jul 2025 17:11:05 +0000 Subject: [PATCH 03/17] Fix lint issue. Add comment --- .../exporter/otlp/proto/common/_internal/__init__.py | 1 + .../tests/test_common.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py index a3fd615680..26a7eeb1b7 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -63,6 +63,7 @@ def filter(self, record): if current_log != getattr(self, "last_log", None): self.last_log = current_log return True + # False means python's `logging` module will no longer process this log. return False diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py index fbf0276158..e803b64121 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py @@ -7,7 +7,7 @@ class TestCommonFuncs(unittest.TestCase): - def test_duplicate_logs_filter_Works(self): + def test_duplicate_logs_filter_works(self): test_logger = logging.getLogger("testLogger") test_logger.addFilter(DuplicateFilter()) with self.assertLogs("testLogger") as cm: From 007de1641ef299d1f1be39af1f9ccf62f3c99725 Mon Sep 17 00:00:00 2001 From: DylanRussell Date: Tue, 22 Jul 2025 13:28:39 +0000 Subject: [PATCH 04/17] Update CHANGELOG.md Co-authored-by: Riccardo Magliocchetti --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c74d05736f..81d09616f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ the OTLP `LogHandler` remains attached to the root logger. Fix a bug that can cause a deadlock to occur over `logging._lock` in some cases ([#4636](https://github.com/open-telemetry/opentelemetry-python/pull/4636)). - Filter duplicate logs emitted from the OTLP exporters to avoid endlessly logging when the OTLP logger itself is failing to export logs. + ([#4695](https://github.com/open-telemetry/opentelemetry-python/pull/4695)). ## Version 1.35.0/0.56b0 (2025-07-11) From 99c7af3ee97dc3c84ac48b559849f840dd2fb3d2 Mon Sep 17 00:00:00 2001 From: DylanRussell Date: Tue, 22 Jul 2025 13:28:59 +0000 Subject: [PATCH 05/17] Update exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py Co-authored-by: Riccardo Magliocchetti --- .../exporter/otlp/proto/common/_internal/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py index 26a7eeb1b7..65f330b989 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -53,6 +53,7 @@ class DuplicateFilter(logging.Filter): + """This prevents logs generated when a log fails to be written to generate another log which fails to be written""" def filter(self, record): current_log = ( record.module, From 03cd086c3d79ed83b37ae22ede7ecde927eb7a5c Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Tue, 22 Jul 2025 19:36:40 +0000 Subject: [PATCH 06/17] Add filter to BatchProcessor class --- .../src/opentelemetry/sdk/_shared_internal/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py index 97a00980cb..2b1d2482ce 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py @@ -27,6 +27,9 @@ Protocol, TypeVar, ) +from opentelemetry.exporter.otlp.proto.common._internal import ( + DuplicateFilter, +) from opentelemetry.context import ( _SUPPRESS_INSTRUMENTATION_KEY, @@ -87,6 +90,7 @@ def __init__( daemon=True, ) self._logger = logging.getLogger(__name__) + self._logger.addFilter(DuplicateFilter()) self._exporting = exporting self._shutdown = False From 49197c99090f2239dd075bb1b45e7d48cb3bf18d Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Tue, 22 Jul 2025 19:54:46 +0000 Subject: [PATCH 07/17] Run precommit and add DuplicateFilter to another place.. --- .../exporter/otlp/proto/common/_internal/__init__.py | 1 + .../opentelemetry/sdk/_logs/_internal/export/__init__.py | 4 ++++ .../src/opentelemetry/sdk/_shared_internal/__init__.py | 6 +++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py index 65f330b989..3cf750a743 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -54,6 +54,7 @@ class DuplicateFilter(logging.Filter): """This prevents logs generated when a log fails to be written to generate another log which fails to be written""" + def filter(self, record): current_log = ( record.module, diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/export/__init__.py index ec629221b8..13ab536afb 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/export/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/export/__init__.py @@ -26,6 +26,9 @@ detach, set_value, ) +from opentelemetry.exporter.otlp.proto.common._internal import ( + DuplicateFilter, +) from opentelemetry.sdk._logs import LogData, LogRecord, LogRecordProcessor from opentelemetry.sdk._shared_internal import BatchProcessor from opentelemetry.sdk.environment_variables import ( @@ -43,6 +46,7 @@ "Unable to parse value for %s as integer. Defaulting to %s." ) _logger = logging.getLogger(__name__) +_logger.addFilter(DuplicateFilter()) class LogExportResult(enum.Enum): diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py index 2b1d2482ce..86dfd026d5 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py @@ -27,9 +27,6 @@ Protocol, TypeVar, ) -from opentelemetry.exporter.otlp.proto.common._internal import ( - DuplicateFilter, -) from opentelemetry.context import ( _SUPPRESS_INSTRUMENTATION_KEY, @@ -37,6 +34,9 @@ detach, set_value, ) +from opentelemetry.exporter.otlp.proto.common._internal import ( + DuplicateFilter, +) from opentelemetry.util._once import Once From cb1a158b74ee7872b6549274685cbd8b5a5a0500 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Wed, 23 Jul 2025 14:51:48 +0000 Subject: [PATCH 08/17] Move DuplicateFilter to SDK --- .../otlp/proto/common/_internal/__init__.py | 18 ---------------- .../tests/test_common.py | 16 -------------- .../exporter/otlp/proto/grpc/exporter.py | 2 +- .../otlp/proto/http/_log_exporter/__init__.py | 4 +--- .../sdk/_logs/_internal/export/__init__.py | 5 +---- .../sdk/_shared_internal/__init__.py | 21 ++++++++++++++++--- .../shared_internal/test_batch_processor.py | 14 +++++++++++++ 7 files changed, 35 insertions(+), 45 deletions(-) delete mode 100644 exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py index 3cf750a743..200644368d 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -16,7 +16,6 @@ from __future__ import annotations import logging -import time from collections.abc import Sequence from typing import ( Any, @@ -52,23 +51,6 @@ _ResourceDataT = TypeVar("_ResourceDataT") -class DuplicateFilter(logging.Filter): - """This prevents logs generated when a log fails to be written to generate another log which fails to be written""" - - def filter(self, record): - current_log = ( - record.module, - record.levelno, - record.msg, - time.time() // 60, - ) - if current_log != getattr(self, "last_log", None): - self.last_log = current_log - return True - # False means python's `logging` module will no longer process this log. - return False - - def _encode_instrumentation_scope( instrumentation_scope: InstrumentationScope, ) -> PB2InstrumentationScope: diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py deleted file mode 100644 index e803b64121..0000000000 --- a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_common.py +++ /dev/null @@ -1,16 +0,0 @@ -import logging -import unittest - -from opentelemetry.exporter.otlp.proto.common._internal import ( - DuplicateFilter, -) - - -class TestCommonFuncs(unittest.TestCase): - def test_duplicate_logs_filter_works(self): - test_logger = logging.getLogger("testLogger") - test_logger.addFilter(DuplicateFilter()) - with self.assertLogs("testLogger") as cm: - test_logger.info("message") - test_logger.info("message") - self.assertEqual(len(cm.output), 1) diff --git a/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py b/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py index fb92d4e0fe..6da3cac134 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/exporter.py @@ -48,7 +48,6 @@ ssl_channel_credentials, ) from opentelemetry.exporter.otlp.proto.common._internal import ( - DuplicateFilter, _get_resource_data, ) from opentelemetry.exporter.otlp.proto.grpc import ( @@ -60,6 +59,7 @@ KeyValue, ) from opentelemetry.proto.resource.v1.resource_pb2 import Resource # noqa: F401 +from opentelemetry.sdk._shared_internal import DuplicateFilter from opentelemetry.sdk.environment_variables import ( OTEL_EXPORTER_OTLP_CERTIFICATE, OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py index 5de62c9f91..bfe207119b 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py @@ -24,9 +24,6 @@ import requests from requests.exceptions import ConnectionError -from opentelemetry.exporter.otlp.proto.common._internal import ( - DuplicateFilter, -) from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs from opentelemetry.exporter.otlp.proto.http import ( _OTLP_HTTP_HEADERS, @@ -40,6 +37,7 @@ LogExporter, LogExportResult, ) +from opentelemetry.sdk._shared_internal import DuplicateFilter from opentelemetry.sdk.environment_variables import ( OTEL_EXPORTER_OTLP_CERTIFICATE, OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/export/__init__.py index 13ab536afb..411f92aec1 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/export/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/export/__init__.py @@ -26,11 +26,8 @@ detach, set_value, ) -from opentelemetry.exporter.otlp.proto.common._internal import ( - DuplicateFilter, -) from opentelemetry.sdk._logs import LogData, LogRecord, LogRecordProcessor -from opentelemetry.sdk._shared_internal import BatchProcessor +from opentelemetry.sdk._shared_internal import BatchProcessor, DuplicateFilter from opentelemetry.sdk.environment_variables import ( OTEL_BLRP_EXPORT_TIMEOUT, OTEL_BLRP_MAX_EXPORT_BATCH_SIZE, diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py index 86dfd026d5..de828d3076 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py @@ -19,6 +19,7 @@ import logging import os import threading +import time import weakref from abc import abstractmethod from typing import ( @@ -34,12 +35,26 @@ detach, set_value, ) -from opentelemetry.exporter.otlp.proto.common._internal import ( - DuplicateFilter, -) from opentelemetry.util._once import Once +class DuplicateFilter(logging.Filter): + """This prevents logs generated when a log fails to be written to generate another log which fails to be written""" + + def filter(self, record): + current_log = ( + record.module, + record.levelno, + record.msg, + time.time() // 60, + ) + if current_log != getattr(self, "last_log", None): + self.last_log = current_log + return True + # False means python's `logging` module will no longer process this log. + return False + + class BatchExportStrategy(enum.Enum): EXPORT_ALL = 0 EXPORT_WHILE_BATCH_EXCEEDS_THRESHOLD = 1 diff --git a/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py b/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py index 4888d81779..6f7d41669c 100644 --- a/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py +++ b/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py @@ -14,6 +14,7 @@ # pylint: disable=protected-access import gc +import logging import multiprocessing import os import time @@ -31,6 +32,9 @@ from opentelemetry.sdk._logs.export import ( BatchLogRecordProcessor, ) +from opentelemetry.sdk._shared_internal import ( + DuplicateFilter, +) from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.sdk.util.instrumentation import InstrumentationScope @@ -193,3 +197,13 @@ def test_record_processor_is_garbage_collected( # Then the reference to the processor should no longer exist assert weak_ref() is None + + +class TestCommonFuncs(unittest.TestCase): + def test_duplicate_logs_filter_works(self): + test_logger = logging.getLogger("testLogger") + test_logger.addFilter(DuplicateFilter()) + with self.assertLogs("testLogger") as cm: + test_logger.info("message") + test_logger.info("message") + self.assertEqual(len(cm.output), 1) From bcfc20c87206107f5bf4e2e2f9025325dddf8dfa Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Wed, 23 Jul 2025 14:56:16 +0000 Subject: [PATCH 09/17] improve changelog entry --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7016892a27..570080816b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Overwrite logging.config.fileConfig and logging.config.dictConfig to ensure the OTLP `LogHandler` remains attached to the root logger. Fix a bug that can cause a deadlock to occur over `logging._lock` in some cases ([#4636](https://github.com/open-telemetry/opentelemetry-python/pull/4636)). -- Filter duplicate logs emitted from the OTLP exporters to avoid endlessly logging when the OTLP logger itself -is failing to export logs. +- Filter duplicate logs out of some internal `logger`'s logs on the export logs path that might otherwise endlessly log or cause a recursion depth exceeded issue in cases where logging itself results in an exception. ([#4695](https://github.com/open-telemetry/opentelemetry-python/pull/4695)). - Update OTLP gRPC/HTTP exporters: calling shutdown will now interrupt exporters that are sleeping From 0caca6b67360554d9eb1b9f3b2f91bca622f4026 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Wed, 23 Jul 2025 14:58:59 +0000 Subject: [PATCH 10/17] Precommit and comment changes --- .../src/opentelemetry/sdk/_shared_internal/__init__.py | 5 ++++- .../tests/shared_internal/test_batch_processor.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py index 0d856542e2..70b07eddc6 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py @@ -40,7 +40,10 @@ class DuplicateFilter(logging.Filter): - """This prevents logs generated when a log fails to be written to generate another log which fails to be written""" + """Filter that can be applied to internal `logger`'s. + + Currently applied to `logger`s on the export logs path that could otherwise cause endless logging of errors or a + recursion depth exceeded issue in cases where logging itself results in an exception.""" def filter(self, record): current_log = ( diff --git a/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py b/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py index 33b0b2786a..34f3992e1e 100644 --- a/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py +++ b/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py @@ -251,6 +251,7 @@ def test_shutdown_allows_1_export_to_finish( assert "Exception while exporting" in caplog.text assert 2 == exporter.num_export_calls + class TestCommonFuncs(unittest.TestCase): def test_duplicate_logs_filter_works(self): test_logger = logging.getLogger("testLogger") @@ -258,4 +259,4 @@ def test_duplicate_logs_filter_works(self): with self.assertLogs("testLogger") as cm: test_logger.info("message") test_logger.info("message") - self.assertEqual(len(cm.output), 1) \ No newline at end of file + self.assertEqual(len(cm.output), 1) From 17ce88b55f2b3b47b823b2eba92f0fc985810992 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Wed, 23 Jul 2025 18:56:56 +0000 Subject: [PATCH 11/17] Fix broken test --- .../tests/shared_internal/test_batch_processor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py b/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py index 34f3992e1e..7cff4b4a22 100644 --- a/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py +++ b/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py @@ -60,6 +60,7 @@ def __init__(self, export_sleep: int): self.num_export_calls = 0 self.export_sleep = export_sleep self._shutdown = False + self.sleep_interrupted = False self.export_sleep_event = threading.Event() def export(self, _: list[Any]): @@ -69,6 +70,7 @@ def export(self, _: list[Any]): sleep_interrupted = self.export_sleep_event.wait(self.export_sleep) if sleep_interrupted: + self.sleep_interrupted = True raise ValueError("Did not get to finish !") def shutdown(self): @@ -248,7 +250,7 @@ def test_shutdown_allows_1_export_to_finish( time.sleep(0.1) assert processor._batch_processor._worker_thread.is_alive() is False # Expect the second call to be interrupted by shutdown, and the third call to never be made. - assert "Exception while exporting" in caplog.text + assert exporter.sleep_interrupted == True assert 2 == exporter.num_export_calls From 614c5c6b0181b39764fee88d143f941b8e033bb2 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Wed, 23 Jul 2025 18:57:56 +0000 Subject: [PATCH 12/17] Precommit --- .../tests/shared_internal/test_batch_processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py b/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py index 7cff4b4a22..f07ebc5ae7 100644 --- a/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py +++ b/opentelemetry-sdk/tests/shared_internal/test_batch_processor.py @@ -225,7 +225,7 @@ def test_record_processor_is_garbage_collected( assert weak_ref() is None def test_shutdown_allows_1_export_to_finish( - self, batch_processor_class, telemetry, caplog + self, batch_processor_class, telemetry ): # This exporter throws an exception if it's export sleep cannot finish. exporter = MockExporterForTesting(export_sleep=2) @@ -250,7 +250,7 @@ def test_shutdown_allows_1_export_to_finish( time.sleep(0.1) assert processor._batch_processor._worker_thread.is_alive() is False # Expect the second call to be interrupted by shutdown, and the third call to never be made. - assert exporter.sleep_interrupted == True + assert exporter.sleep_interrupted is True assert 2 == exporter.num_export_calls From 470ff1bbc7ba3a3994de00769c4b65d938529597 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Wed, 23 Jul 2025 19:30:31 +0000 Subject: [PATCH 13/17] Fix lint issue --- .../src/opentelemetry/sdk/_shared_internal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py index 70b07eddc6..f458837329 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py @@ -53,7 +53,7 @@ def filter(self, record): time.time() // 60, ) if current_log != getattr(self, "last_log", None): - self.last_log = current_log + self.last_log = current_log # pylint: disable=attribute-defined-outside-init return True # False means python's `logging` module will no longer process this log. return False From 95e10b6928cb5bbfb45ca94d02d9a5dc52e9b645 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 24 Jul 2025 13:18:42 +0000 Subject: [PATCH 14/17] precommit --- .../src/opentelemetry/sdk/_shared_internal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py index f458837329..235a6737c0 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py @@ -53,7 +53,7 @@ def filter(self, record): time.time() // 60, ) if current_log != getattr(self, "last_log", None): - self.last_log = current_log # pylint: disable=attribute-defined-outside-init + self.last_log = current_log # pylint: disable=attribute-defined-outside-init return True # False means python's `logging` module will no longer process this log. return False From 9c817511fae5e0438a5ddd5d6bb0bb2d7eb94487 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 24 Jul 2025 18:32:05 +0000 Subject: [PATCH 15/17] test repro of issue --- repro.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 repro.py diff --git a/repro.py b/repro.py new file mode 100644 index 0000000000..adc8fe9bb3 --- /dev/null +++ b/repro.py @@ -0,0 +1,46 @@ +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "fastapi", +# "opentelemetry-distro", +# "opentelemetry-instrumentation", +# "opentelemetry-exporter-otlp", +# "uvicorn", +# "opentelemetry-instrumentation-fastapi", +# "opentelemetry-instrumentation-asgi", +# "opentelemetry-util-http", +# "opentelemetry-semantic-conventions", +# ] +# /// + +import os +import logging +logging.basicConfig(level=0) +os.environ["OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"] = "true" + +from opentelemetry.instrumentation import auto_instrumentation +from opentelemetry._logs import ( + NoOpLogger, + SeverityNumber, + get_logger, + get_logger_provider, +) +auto_instrumentation.initialize(swallow_exceptions=False) + +import uvicorn + +from fastapi import FastAPI + +app = FastAPI() +print(logging.root.handlers) +@app.get("/") +async def root(): + logging.info("Handling request for root endpoint") + return {"message": "Hello World"} + +logging.info("AGJAJSGJAG") + +uvicorn.run(app, host="0.0.0.0", port=3000) +print("RUNNING !") +# provider = get_logger_provider() +# provider.shutdown() From 3cdd4d14ab47c1cba59b6a0d7c25c0137927d0f4 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 24 Jul 2025 20:38:17 +0000 Subject: [PATCH 16/17] add print statements --- .../src/opentelemetry/sdk/_configuration/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/__init__.py index 60640739e3..36b6fd9457 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/__init__.py @@ -255,6 +255,7 @@ def _init_logging( setup_logging_handler: bool = True, exporter_args_map: ExporterArgsMap | None = None, ): + print("in init logging") provider = LoggerProvider(resource=resource) set_logger_provider(provider) @@ -269,6 +270,7 @@ def _init_logging( set_event_logger_provider(event_logger_provider) if setup_logging_handler: + print("setting up handler..") # Add OTel handler handler = LoggingHandler( level=logging.NOTSET, logger_provider=provider @@ -417,6 +419,7 @@ def _initialize_components( setup_logging_handler: bool | None = None, exporter_args_map: ExporterArgsMap | None = None, ): + print("initializing components") if trace_exporter_names is None: trace_exporter_names = [] if metric_exporter_names is None: From e6867e0b4f59994ee45aee1f6381bf33a1f90f3b Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Fri, 25 Jul 2025 15:07:46 +0000 Subject: [PATCH 17/17] undo debug stuff --- .../sdk/_configuration/__init__.py | 3 -- repro.py | 46 ------------------- 2 files changed, 49 deletions(-) delete mode 100644 repro.py diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/__init__.py index 36b6fd9457..60640739e3 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_configuration/__init__.py @@ -255,7 +255,6 @@ def _init_logging( setup_logging_handler: bool = True, exporter_args_map: ExporterArgsMap | None = None, ): - print("in init logging") provider = LoggerProvider(resource=resource) set_logger_provider(provider) @@ -270,7 +269,6 @@ def _init_logging( set_event_logger_provider(event_logger_provider) if setup_logging_handler: - print("setting up handler..") # Add OTel handler handler = LoggingHandler( level=logging.NOTSET, logger_provider=provider @@ -419,7 +417,6 @@ def _initialize_components( setup_logging_handler: bool | None = None, exporter_args_map: ExporterArgsMap | None = None, ): - print("initializing components") if trace_exporter_names is None: trace_exporter_names = [] if metric_exporter_names is None: diff --git a/repro.py b/repro.py deleted file mode 100644 index adc8fe9bb3..0000000000 --- a/repro.py +++ /dev/null @@ -1,46 +0,0 @@ -# /// script -# requires-python = ">=3.13" -# dependencies = [ -# "fastapi", -# "opentelemetry-distro", -# "opentelemetry-instrumentation", -# "opentelemetry-exporter-otlp", -# "uvicorn", -# "opentelemetry-instrumentation-fastapi", -# "opentelemetry-instrumentation-asgi", -# "opentelemetry-util-http", -# "opentelemetry-semantic-conventions", -# ] -# /// - -import os -import logging -logging.basicConfig(level=0) -os.environ["OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"] = "true" - -from opentelemetry.instrumentation import auto_instrumentation -from opentelemetry._logs import ( - NoOpLogger, - SeverityNumber, - get_logger, - get_logger_provider, -) -auto_instrumentation.initialize(swallow_exceptions=False) - -import uvicorn - -from fastapi import FastAPI - -app = FastAPI() -print(logging.root.handlers) -@app.get("/") -async def root(): - logging.info("Handling request for root endpoint") - return {"message": "Hello World"} - -logging.info("AGJAJSGJAG") - -uvicorn.run(app, host="0.0.0.0", port=3000) -print("RUNNING !") -# provider = get_logger_provider() -# provider.shutdown()