This is an automated email from the ASF dual-hosted git repository. baunsgaard pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/systemds.git
commit be149492ac57578871d6f65280de716bdef4be8b Author: baunsgaard <[email protected]> AuthorDate: Thu Nov 4 10:30:53 2021 +0100 [SYSTEMDS-3199] Python more intuitive error messages When encountering an error in python a very bloated stack trace is printed along with std out and err from the jvm. This also happens in cases where compute is called when the JVM is down. To reduce this bloat, and make it more understandable this commit change the error message when the JVM is not available to a small message. and remove the std out and err if they are empty. --- .../python/systemds/context/systemds_context.py | 21 +++++++++++-------- src/main/python/systemds/script_building/script.py | 24 +++++++++++++++------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/main/python/systemds/context/systemds_context.py b/src/main/python/systemds/context/systemds_context.py index eb65887..afa38c2 100644 --- a/src/main/python/systemds/context/systemds_context.py +++ b/src/main/python/systemds/context/systemds_context.py @@ -21,14 +21,12 @@ __all__ = ["SystemDSContext"] -import copy import json import os import socket -import threading -import time +import sys from glob import glob -from queue import Empty, Queue +from queue import Queue from subprocess import PIPE, Popen from threading import Thread from time import sleep @@ -105,7 +103,7 @@ class SystemDSContext(object): else: return [self.__stderr.get() for x in range(lines)] - def exception_and_close(self, e: Exception): + def exception_and_close(self, exception_str: str, trace_back_limit : int = None): """ Method for printing exception, printing stdout and error, while also closing the context correctly. @@ -113,10 +111,15 @@ class SystemDSContext(object): """ # e = sys.exc_info()[0] - message = "Exception Encountered! closing JVM\n" - message += "standard out :\n" + "\n".join(self.get_stdout()) - message += "standard error :\n" + "\n".join(self.get_stdout()) - message += "Exception : " + str(e) + message = "" + stdOut = self.get_stdout() + if stdOut: + message += "standard out :\n" + "\n".join(stdOut) + stdErr = self.get_stderr() + if stdErr: + message += "standard error :\n" + "\n".join(stdErr) + message += exception_str + sys.tracebacklimit = trace_back_limit self.close() raise RuntimeError(message) diff --git a/src/main/python/systemds/script_building/script.py b/src/main/python/systemds/script_building/script.py index 318292b..313c6b9 100644 --- a/src/main/python/systemds/script_building/script.py +++ b/src/main/python/systemds/script_building/script.py @@ -19,11 +19,12 @@ # # ------------------------------------------------------------- -from typing import Any, Collection, KeysView, Tuple, Union, Optional, Dict, TYPE_CHECKING, List +from typing import (TYPE_CHECKING, Any, Collection, Dict, KeysView, List, + Optional, Tuple, Union) +from py4j.protocol import Py4JNetworkError from py4j.java_collections import JavaArray -from py4j.java_gateway import JavaObject, JavaGateway - +from py4j.java_gateway import JavaGateway, JavaObject from systemds.script_building.dag import DAGNode, OutputType from systemds.utils.consts import VALID_INPUT_TYPES @@ -79,9 +80,14 @@ class DMLScript: self.__prepare_script() ret = self.prepared_script.executeScript() return ret + except Py4JNetworkError: + exception_str = "Py4JNetworkError: no connection to JVM, most likely due to previous crash" + trace_back_limit = 0 except Exception as e: - self.sds_context.exception_and_close(e) - return None + exception_str = str(e) + trace_back_limit = None + self.sds_context.exception_and_close(exception_str, trace_back_limit) + def execute_with_lineage(self) -> Tuple[JavaObject, str]: """If not already created, create a preparedScript from our DMLCode, pass python local data to our prepared @@ -104,9 +110,13 @@ class DMLScript: traces.append(self.prepared_script.getLineageTrace(output)) return ret, traces + except Py4JNetworkError: + exception_str = "Py4JNetworkError: no connection to JVM, most likely due to previous crash" + trace_back_limit = 0 except Exception as e: - self.sds_context.exception_and_close(e) - return None, None + exception_str = str(e) + trace_back_limit = None + self.sds_context.exception_and_close(exception_str, trace_back_limit) def __prepare_script(self): gateway = self.sds_context.java_gateway
