allisonwang-db commented on code in PR #41927:
URL: https://github.com/apache/spark/pull/41927#discussion_r1261736414
##########
python/pyspark/errors/error_classes.py:
##########
@@ -171,9 +171,7 @@
},
"DIFFERENT_SCHEMA" : {
"message" : [
- "Schemas do not match:",
- "df schema: <df_schema>",
- "expected schema: <expected_schema>"
+ "<error_msg>"
Review Comment:
Seems like we can have `Schemas do not match` in the error message here.
##########
python/pyspark/testing/utils.py:
##########
@@ -221,7 +221,127 @@ def check_error(
)
-def assertDataFrameEqual(df: DataFrame, expected: DataFrame, check_row_order:
bool = False):
+def assertSchemaEqual(actual: StructType, expected: StructType,
check_nullability: bool = True):
Review Comment:
Can the expected schema be a string, like `a: int, b: int`?
##########
python/pyspark/testing/utils.py:
##########
@@ -221,7 +221,127 @@ def check_error(
)
-def assertDataFrameEqual(df: DataFrame, expected: DataFrame, check_row_order:
bool = False):
+def assertSchemaEqual(actual: StructType, expected: StructType,
check_nullability: bool = True):
+ """
+ A util function to assert equality between DataFrame schemas `actual`
+ and `expected`, with optional parameter `check_nullability`.
+
+ .. versionadded:: 3.5.0
+
+ Parameters
+ ----------
+ actual : StructType
+ The DataFrame schema that is being compared or tested.
+ expected : StructType
+ The expected schema, for comparison with the actual schema.
+ check_nullability : bool, optional
+ A flag indicating whether the nullable flag should be ignored in
schema comparison.
+ If set to `False`, the nullable flag in the schemas is not taken into
account.
+ If set to `True` (default), the nullable flag will be checked during
schema comparison.
+
+ Examples
+ --------
+ >>> from pyspark.sql.types import StructType, StructField, ArrayType,
IntegerType, DoubleType
+ >>> s1 = StructType([StructField("names", ArrayType(DoubleType(), True),
True)])
+ >>> s2 = StructType([StructField("names", ArrayType(DoubleType(), True),
True)])
+ >>> assertSchemaEqual(s1, s2) # pass
+ >>> s1 = StructType([StructField("names", ArrayType(IntegerType(), True),
True)])
+ >>> s2 = StructType([StructField("names", ArrayType(DoubleType(), False),
True)])
+ >>> assertSchemaEqual(s1, s2) # fail # doctest: +IGNORE_EXCEPTION_DETAIL
+NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ PySparkAssertionError: [DIFFERENT_SCHEMA] Schemas do not match:
+ [df]
+ StructField("names", ArrayType(IntegerType(), True), True)
+
+ [expected]
+ StructField("names", ArrayType(DoubleType(), False), True)
+
+ """
+
+ def compare_schemas_ignore_nullable(s1, s2):
+ if len(s1) != len(s2):
+ return False
+ zipped = zip_longest(s1, s2)
+ for sf1, sf2 in zipped:
+ if not compare_structfields_ignore_nullable(sf1, sf2):
+ return False
+ return True
+
+ def compare_structfields_ignore_nullable(actualSF, expectedSF):
+ if not check_nullability:
+ if actualSF is None and expectedSF is None:
+ return True
+ elif actualSF is None or expectedSF is None:
+ return False
+ if actualSF.name != expectedSF.name:
+ return False
+ else:
+ return compare_datatypes_ignore_nullable(actualSF.dataType,
expectedSF.dataType)
+ else:
+ return actualSF == expectedSF
+
+ def compare_datatypes_ignore_nullable(dt1, dt2):
+ # checks datatype equality, using recursion to ignore nullable
+ if dt1.typeName() == dt2.typeName():
+ if dt1.typeName() == "array":
+ return compare_datatypes_ignore_nullable(dt1.elementType,
dt2.elementType)
+ elif dt1.typeName() == "struct":
+ return compare_schemas_ignore_nullable(dt1, dt2)
+ else:
+ return True
+ else:
+ return False
+
+ schemas_equal = True
+ error_msg = "Schemas do not match: \n"
+
+ if not check_nullability:
+ if not compare_schemas_ignore_nullable(actual, expected):
+ zipped = zip_longest(actual, expected)
+ for actualSF, expectedSF in zipped:
+ if not compare_structfields_ignore_nullable(actualSF,
expectedSF):
+ schemas_equal = False
+ error_msg += (
+ "[df]"
+ + "\n"
+ + str(actualSF)
+ + "\n\n"
+ + "[expected]"
+ + "\n"
+ + str(expectedSF)
+ + "\n\n"
+ )
+ else:
+ if actual != expected:
Review Comment:
Should we also check if both actual and expected are StructType?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
