Repository: spark Updated Branches: refs/heads/branch-2.3 9db81fd86 -> 31dab7140
[SPARK-25072][PYSPARK] Forbid extra value for custom Row ## What changes were proposed in this pull request? Add value length check in `_create_row`, forbid extra value for custom Row in PySpark. ## How was this patch tested? New UT in pyspark-sql Closes #22140 from xuanyuanking/SPARK-25072. Lead-authored-by: liyuanjian <liyuanj...@baidu.com> Co-authored-by: Yuanjian Li <xyliyuanj...@gmail.com> Signed-off-by: Bryan Cutler <cutl...@gmail.com> (cherry picked from commit c84bc40d7f33c71eca1c08f122cd60517f34c1f8) Signed-off-by: Bryan Cutler <cutl...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/31dab714 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/31dab714 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/31dab714 Branch: refs/heads/branch-2.3 Commit: 31dab7140a4b271e7b976762af7a36f8bfbb8381 Parents: 9db81fd Author: liyuanjian <liyuanj...@baidu.com> Authored: Thu Sep 6 10:17:29 2018 -0700 Committer: Bryan Cutler <cutl...@gmail.com> Committed: Thu Sep 6 10:18:04 2018 -0700 ---------------------------------------------------------------------- python/pyspark/sql/tests.py | 4 ++++ python/pyspark/sql/types.py | 3 +++ 2 files changed, 7 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/31dab714/python/pyspark/sql/tests.py ---------------------------------------------------------------------- diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index 6bfb329..374db54 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -268,6 +268,10 @@ class DataTypeTests(unittest.TestCase): struct_field = StructField("a", IntegerType()) self.assertRaises(TypeError, struct_field.typeName) + def test_invalid_create_row(self): + row_class = Row("c1", "c2") + self.assertRaises(ValueError, lambda: row_class(1, 2, 3)) + class SQLTests(ReusedSQLTestCase): http://git-wip-us.apache.org/repos/asf/spark/blob/31dab714/python/pyspark/sql/types.py ---------------------------------------------------------------------- diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index cd85740..8a8cd8d 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -1532,6 +1532,9 @@ class Row(tuple): # let object acts like class def __call__(self, *args): """create new Row object""" + if len(args) > len(self): + raise ValueError("Can not create Row with fields %s, expected %d values " + "but got %s" % (self, len(self), args)) return _create_row(self, args) def __getitem__(self, item): --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
