This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 9eb45ec [SPARK-35408][PYTHON] Improve parameter validation in DataFrame.show 9eb45ec is described below commit 9eb45ecb4f39f372e20529da468f304c4ec7c175 Author: Gera Shegalov <g...@apache.org> AuthorDate: Mon May 17 16:22:46 2021 +0900 [SPARK-35408][PYTHON] Improve parameter validation in DataFrame.show ### What changes were proposed in this pull request? Provide clearer error message tied to the user's Python code if incorrect parameters are passed to `DataFrame.show` rather than the message about a missing JVM method the user is not calling directly. ``` py4j.Py4JException: Method showString([class java.lang.Boolean, class java.lang.Integer, class java.lang.Boolean]) does not exist at py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:318) at py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:326) at py4j.Gateway.invoke(Gateway.java:274) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:238) at java.lang.Thread.run(Thread.java:748 ``` ### Why are the changes needed? For faster debugging through actionable error message. ### Does this PR introduce _any_ user-facing change? No change for the correct parameters but different error messages for the parameters triggering an exception. ### How was this patch tested? - unit test - manually in PySpark REPL Closes #32555 from gerashegalov/df_show_validation. Authored-by: Gera Shegalov <g...@apache.org> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- python/pyspark/sql/dataframe.py | 16 ++++++++++++++-- python/pyspark/sql/tests/test_dataframe.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 8fe263e..22cc7a4 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -448,7 +448,7 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin): ---------- n : int, optional Number of rows to show. - truncate : bool, optional + truncate : bool or int, optional If set to ``True``, truncate strings longer than 20 chars by default. If set to a number greater than one, truncates long strings to length ``truncate`` and align cells right. @@ -482,10 +482,22 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin): age | 5 name | Bob """ + + if not isinstance(n, int) or isinstance(n, bool): + raise TypeError("Parameter 'n' (number of rows) must be an int") + + if not isinstance(vertical, bool): + raise TypeError("Parameter 'vertical' must be a bool") + if isinstance(truncate, bool) and truncate: print(self._jdf.showString(n, 20, vertical)) else: - print(self._jdf.showString(n, int(truncate), vertical)) + try: + int_truncate = int(truncate) + except ValueError: + raise TypeError(f"Parameter 'truncate={truncate}' should be either bool or int.") + + print(self._jdf.showString(n, int_truncate, vertical)) def __repr__(self): if not self._support_repr_html and self.sql_ctx._conf.isReplEagerEvalEnabled(): diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py index 3e961cb..74895c0 100644 --- a/python/pyspark/sql/tests/test_dataframe.py +++ b/python/pyspark/sql/tests/test_dataframe.py @@ -837,6 +837,24 @@ class DataFrameTests(ReusedSQLTestCase): finally: shutil.rmtree(tpath) + def test_df_show(self): + # SPARK-35408: ensure better diagnostics if incorrect parameters are passed + # to DataFrame.show + + df = self.spark.createDataFrame([('foo',)]) + df.show(5) + df.show(5, True) + df.show(5, 1, True) + df.show(n=5, truncate='1', vertical=False) + df.show(n=5, truncate=1.5, vertical=False) + + with self.assertRaisesRegex(TypeError, "Parameter 'n'"): + df.show(True) + with self.assertRaisesRegex(TypeError, "Parameter 'vertical'"): + df.show(vertical='foo') + with self.assertRaisesRegex(TypeError, "Parameter 'truncate=foo'"): + df.show(truncate='foo') + class QueryExecutionListenerTests(unittest.TestCase, SQLTestUtils): # These tests are separate because it uses 'spark.sql.queryExecutionListeners' which is --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org