[
https://issues.apache.org/jira/browse/SPARK-41901?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Apache Spark reassigned SPARK-41901:
------------------------------------
Assignee: (was: Apache Spark)
> Parity in String representation of Column
> -----------------------------------------
>
> Key: SPARK-41901
> URL: https://issues.apache.org/jira/browse/SPARK-41901
> Project: Spark
> Issue Type: Sub-task
> Components: Connect
> Affects Versions: 3.4.0
> Reporter: Sandeep Singh
> Priority: Major
>
> {code:java}
> from pyspark.sql import functions
> funs = [
> (functions.acosh, "ACOSH"),
> (functions.asinh, "ASINH"),
> (functions.atanh, "ATANH"),
> ]
> cols = ["a", functions.col("a")]
> for f, alias in funs:
> for c in cols:
> self.assertIn(f"{alias}(a)", repr(f(c))){code}
> {code:java}
> Traceback (most recent call last):
> File
> "/Users/s.singh/personal/spark-oss/python/pyspark/sql/tests/test_functions.py",
> line 271, in test_inverse_trig_functions
> self.assertIn(f"{alias}(a)", repr(f(c)))
> AssertionError: 'ACOSH(a)' not found in
> "Column<'acosh(ColumnReference(a))'>"{code}
>
>
> {code:java}
> from pyspark.sql.functions import col, lit, overlay
> from itertools import chain
> import re
> actual = list(
> chain.from_iterable(
> [
> re.findall("(overlay\\(.*\\))", str(x))
> for x in [
> overlay(col("foo"), col("bar"), 1),
> overlay("x", "y", 3),
> overlay(col("x"), col("y"), 1, 3),
> overlay("x", "y", 2, 5),
> overlay("x", "y", lit(11)),
> overlay("x", "y", lit(2), lit(5)),
> ]
> ]
> )
> )
> expected = [
> "overlay(foo, bar, 1, -1)",
> "overlay(x, y, 3, -1)",
> "overlay(x, y, 1, 3)",
> "overlay(x, y, 2, 5)",
> "overlay(x, y, 11, -1)",
> "overlay(x, y, 2, 5)",
> ]
> self.assertListEqual(actual, expected)
> df = self.spark.createDataFrame([("SPARK_SQL", "CORE", 7, 0)], ("x", "y",
> "pos", "len"))
> exp = [Row(ol="SPARK_CORESQL")]
> self.assertTrue(
> all(
> [
> df.select(overlay(df.x, df.y, 7, 0).alias("ol")).collect() == exp,
> df.select(overlay(df.x, df.y, lit(7),
> lit(0)).alias("ol")).collect() == exp,
> df.select(overlay("x", "y", "pos", "len").alias("ol")).collect()
> == exp,
> ]
> )
> ) {code}
> {code:java}
> Traceback (most recent call last):
> File
> "/Users/s.singh/personal/spark-oss/python/pyspark/sql/tests/test_functions.py",
> line 675, in test_overlay
> self.assertListEqual(actual, expected)
> AssertionError: Lists differ: ['overlay(ColumnReference(foo),
> ColumnReference(bar[402 chars]5))'] != ['overlay(foo, bar, 1, -1)',
> 'overlay(x, y, 3, -1)'[90 chars] 5)']
> First differing element 0:
> 'overlay(ColumnReference(foo), ColumnReference(bar), Literal(1), Literal(-1))'
> 'overlay(foo, bar, 1, -1)'
> - ['overlay(ColumnReference(foo), ColumnReference(bar), Literal(1),
> Literal(-1))',
> - 'overlay(ColumnReference(x), ColumnReference(y), Literal(3), Literal(-1))',
> - 'overlay(ColumnReference(x), ColumnReference(y), Literal(1), Literal(3))',
> - 'overlay(ColumnReference(x), ColumnReference(y), Literal(2), Literal(5))',
> - 'overlay(ColumnReference(x), ColumnReference(y), Literal(11),
> Literal(-1))',
> - 'overlay(ColumnReference(x), ColumnReference(y), Literal(2), Literal(5))']
> + ['overlay(foo, bar, 1, -1)',
> + 'overlay(x, y, 3, -1)',
> + 'overlay(x, y, 1, 3)',
> + 'overlay(x, y, 2, 5)',
> + 'overlay(x, y, 11, -1)',
> + 'overlay(x, y, 2, 5)']
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
