This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9c7f613f0f1 [SPARK-44558][CONNECT][PYTHON] Export Spark Log Level
9c7f613f0f1 is described below
commit 9c7f613f0f10f4ee90bb29920157a32017696a43
Author: Alice Sayutina <alice.sayut...@databricks.com>
AuthorDate: Fri Jul 28 11:34:16 2023 +0900
[SPARK-44558][CONNECT][PYTHON] Export Spark Log Level
### What changes were proposed in this pull request?
Export Spark Connect Log Level in pyspark.
### Why are the changes needed?
This is convenient for software dependent on spark-connect so it can be
possible to enable debug logging just in one place.
### Does this PR introduce _any_ user-facing change?
New api is suggested.
### How was this patch tested?
Checked it works from shell
Closes #42175 from cdkrot/spark_connect_debug_logs.
Lead-authored-by: Alice Sayutina <alice.sayut...@databricks.com>
Co-authored-by: Alice Sayutina <cdkr...@gmail.com>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
python/pyspark/sql/connect/client/core.py | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/sql/connect/client/core.py
b/python/pyspark/sql/connect/client/core.py
index 473a00f4001..0288bbc6508 100644
--- a/python/pyspark/sql/connect/client/core.py
+++ b/python/pyspark/sql/connect/client/core.py
@@ -17,6 +17,7 @@
__all__ = [
"ChannelBuilder",
"SparkConnectClient",
+ "getLogLevel",
]
from pyspark.sql.connect.utils import check_dependencies
@@ -104,7 +105,7 @@ def _configure_logging() -> logging.Logger:
# Check the environment variables for log levels:
if "SPARK_CONNECT_LOG_LEVEL" in os.environ:
- logger.setLevel(os.getenv("SPARK_CONNECT_LOG_LEVEL", "error").upper())
+ logger.setLevel(os.environ["SPARK_CONNECT_LOG_LEVEL"].upper())
else:
logger.disabled = True
return logger
@@ -114,6 +115,20 @@ def _configure_logging() -> logging.Logger:
logger = _configure_logging()
+def getLogLevel() -> Optional[int]:
+ """
+ This returns this log level as integer, or none (if no logging is enabled).
+
+ Spark Connect logging can be configured with environment variable
'SPARK_CONNECT_LOG_LEVEL'
+
+ .. versionadded:: 3.5.0
+ """
+
+ if not logger.disabled:
+ return logger.level
+ return None
+
+
class ChannelBuilder:
"""
This is a helper class that is used to create a GRPC channel based on the
given
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
