This is an automated email from the ASF dual-hosted git repository.
estrauss pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 455a73882c [MINOR] PythonAPI: Fix for correct logging setup for debug
455a73882c is described below
commit 455a73882c42246deeb4c2659952bd4729756e18
Author: e-strauss <[email protected]>
AuthorDate: Wed Jul 23 15:32:47 2025 +0200
[MINOR] PythonAPI: Fix for correct logging setup for debug
Closes #2295
---
.../python/systemds/context/systemds_context.py | 57 +++++++++++++++-----
.../python/tests/basics/test_context_creation.py | 61 ++++++++++++++++++++++
2 files changed, 106 insertions(+), 12 deletions(-)
diff --git a/src/main/python/systemds/context/systemds_context.py
b/src/main/python/systemds/context/systemds_context.py
index 25824bc663..a559850c33 100644
--- a/src/main/python/systemds/context/systemds_context.py
+++ b/src/main/python/systemds/context/systemds_context.py
@@ -66,6 +66,7 @@ class SystemDSContext(object):
_log: logging.Logger
__stdout: Queue = None
__stderr: Queue = None
+ _logging_initialized = False
def __init__(
self,
@@ -779,21 +780,53 @@ class SystemDSContext(object):
:param level: The SystemDS logging part logging level.
:param py4j_level: The Py4J logging level.
"""
- logging.basicConfig()
+ # Set py4j level every time
py4j = logging.getLogger("py4j.java_gateway")
py4j.setLevel(py4j_level)
py4j.propagate = False
+ if not SystemDSContext._logging_initialized:
+ # Add handler only once
+ logging.basicConfig()
+
+ root_logger = logging.getLogger(self.__class__.__name__)
+ root_logger.handlers.clear()
+
+ f_handler = DynamicStderrHandler()
+ f_handler.setLevel(logging.NOTSET)
+ f_handler.setFormatter(
+ logging.Formatter(
+ "%(asctime)s %(levelname)s %(name)s: %(message)s",
+ "%y/%m/%d %H:%M:%S",
+ )
+ )
+
+ root_logger.addHandler(f_handler)
+ root_logger.propagate = False
+
+ SystemDSContext._logging_initialized = True
+
+ # Per-instance logger setup
self._log = logging.getLogger(self.__class__.__name__)
- f_handler = logging.StreamHandler()
- f_handler.setLevel(level)
- f_format = logging.Formatter(
- "%(asctime)s - SystemDS- %(levelname)s - %(message)s"
+ self._log.setLevel(level)
+ self._log.debug(
+ "Logging setup done (SystemDS level: %s, Py4J level: %s)",
+ logging.getLevelName(level),
+ logging.getLevelName(py4j_level),
)
- f_handler.setFormatter(f_format)
- self._log.addHandler
- # avoid the logger to call loggers above.
- self._log.propagate = False
- # Reset all handlers to only this new handler.
- self._log.handlers = [f_handler]
- self._log.debug("Logging setup done")
+
+
+class DynamicStderrHandler(logging.StreamHandler):
+ def __init__(self, level=logging.NOTSET, formatter=None):
+ # Avoid setting stream directly to sys.stderr, we will do that
dynamically
+ # For more info see test case:
tests/basics/test_context_creation/test_random_port_debug3
+ # where we redirect the err channel in between different contexts
+ super().__init__(stream=None)
+ self.setLevel(level)
+ if formatter:
+ self.setFormatter(formatter)
+
+ def emit(self, record):
+ # Always use the current sys.stderr when emitting
+ self.stream = sys.stderr
+ super().emit(record)
diff --git a/src/main/python/tests/basics/test_context_creation.py
b/src/main/python/tests/basics/test_context_creation.py
index 920dace8be..3d80420d4e 100644
--- a/src/main/python/tests/basics/test_context_creation.py
+++ b/src/main/python/tests/basics/test_context_creation.py
@@ -21,12 +21,73 @@
import unittest
import logging
+import io
+import sys
+from contextlib import redirect_stdout, redirect_stderr
from systemds.context import SystemDSContext
class TestContextCreation(unittest.TestCase):
+ def test_random_port_debug(self):
+ SystemDSContext._logging_initialized = False
+
+ stderr_buffer = io.StringIO()
+
+ with redirect_stderr(stderr_buffer):
+ sds1 = SystemDSContext(logging_level=10)
+ sds1.close()
+
+ err = stderr_buffer.getvalue()
+ print("Captured STDERR:\n", err)
+ print("END OF STDERR\n")
+
+ self.assertIn("DEBUG SystemDSContext: Logging setup done", err)
+
+ def test_random_port_debug2(self):
+ SystemDSContext._logging_initialized = False
+
+ stderr_buffer = io.StringIO()
+
+ with redirect_stderr(stderr_buffer):
+ sds1 = SystemDSContext()
+ sds1.close()
+
+ err = stderr_buffer.getvalue()
+ print("\nCaptured STDERR (ctx1):\n", err)
+ print("END OF STDERR\n")
+
+ # clear the buffer
+ stderr_buffer.seek(0)
+ stderr_buffer.truncate(0)
+
+ sds2 = SystemDSContext(logging_level=10)
+ sds2.close()
+
+ err = stderr_buffer.getvalue()
+ print("\nCaptured STDERR (ctx2):\n", err)
+ print("END OF STDERR\n")
+
+ self.assertIn("DEBUG SystemDSContext: Logging setup done", err)
+
+ def test_random_port_debug3(self):
+ SystemDSContext._logging_initialized = False
+
+ sds1 = SystemDSContext()
+ sds1.close()
+ stderr_buffer = io.StringIO()
+
+ with redirect_stderr(stderr_buffer):
+ sds2 = SystemDSContext(logging_level=10)
+ sds2.close()
+
+ err = stderr_buffer.getvalue()
+ print("\nCaptured STDERR (ctx2):\n", err)
+ print("END OF STDERR\n")
+
+ self.assertIn("DEBUG SystemDSContext: Logging setup done", err)
+
def test_random_port(self):
sds1 = SystemDSContext()
sds1.close()
