This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6d32472b2a42 [SPARK-49112][CONNECT][TEST] Make
`createLocalRelationProto` support `TimestampType`
6d32472b2a42 is described below
commit 6d32472b2a4225bd847953a679d3be2b077d212e
Author: Ruifeng Zheng <ruife...@apache.org>
AuthorDate: Tue Aug 6 10:00:08 2024 +0900
[SPARK-49112][CONNECT][TEST] Make `createLocalRelationProto` support
`TimestampType`
### What changes were proposed in this pull request?
Make `createLocalRelationProto` support relation with `TimestampType`
### Why are the changes needed?
existing helper function `createLocalRelationProto` cannot create table
with `TimestampType`:
```
org.apache.spark.SparkException: [INTERNAL_ERROR] Missing timezoneId
where it is mandatory. SQLSTATE: XX000
at org.apache.spark.SparkException$.internalError(SparkException.scala:99)
at
org.apache.spark.SparkException$.internalError(SparkException.scala:103)
at org.apache.spark.sql.util.ArrowUtils$.toArrowType(ArrowUtils.scala:57)
at
org.apache.spark.sql.util.ArrowUtils$.toArrowField(ArrowUtils.scala:139)
at
org.apache.spark.sql.util.ArrowUtils$.$anonfun$toArrowSchema$1(ArrowUtils.scala:181)
at
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
```
### Does this PR introduce _any_ user-facing change?
No, test-only
### How was this patch tested?
added ut
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #47608 from zhengruifeng/create_timestamp_localrel.
Authored-by: Ruifeng Zheng <ruife...@apache.org>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
.../sql/connect/planner/SparkConnectPlannerSuite.scala | 5 +++--
.../sql/connect/planner/SparkConnectProtoSuite.scala | 18 +++++++++++++++++-
2 files changed, 20 insertions(+), 3 deletions(-)
diff --git
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala
index 86c10b834f31..e44d3eacc66d 100644
---
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala
+++
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala
@@ -87,7 +87,8 @@ trait SparkConnectPlanTest extends SharedSparkSession {
*/
def createLocalRelationProto(
attrs: Seq[AttributeReference],
- data: Seq[InternalRow]): proto.Relation = {
+ data: Seq[InternalRow],
+ timeZoneId: String = "UTC"): proto.Relation = {
val localRelationBuilder = proto.LocalRelation.newBuilder()
val bytes = ArrowConverters
@@ -96,7 +97,7 @@ trait SparkConnectPlanTest extends SharedSparkSession {
DataTypeUtils.fromAttributes(attrs.map(_.toAttribute)),
Long.MaxValue,
Long.MaxValue,
- null,
+ timeZoneId,
true)
.next()
diff --git
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectProtoSuite.scala
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectProtoSuite.scala
index 190f8cde16f5..03bf5a4c10db 100644
---
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectProtoSuite.scala
+++
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectProtoSuite.scala
@@ -42,7 +42,7 @@ import org.apache.spark.sql.connector.catalog.{Identifier,
InMemoryTableCatalog,
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.CatalogHelper
import org.apache.spark.sql.execution.arrow.ArrowConverters
import org.apache.spark.sql.functions._
-import org.apache.spark.sql.types.{ArrayType, BooleanType, ByteType, DataType,
DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType,
Metadata, ShortType, StringType, StructField, StructType}
+import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
@@ -64,6 +64,11 @@ class SparkConnectProtoSuite extends PlanTest with
SparkConnectPlanTest {
Seq(AttributeReference("id", IntegerType)(), AttributeReference("name",
StringType)()),
Seq.empty)
+ lazy val connectTestRelation3 =
+ createLocalRelationProto(
+ Seq(AttributeReference("id", IntegerType)(), AttributeReference("date",
TimestampType)()),
+ Seq.empty)
+
lazy val connectTestRelationMap =
createLocalRelationProto(
Seq(AttributeReference("id", MapType(StringType, StringType))()),
@@ -79,6 +84,11 @@ class SparkConnectProtoSuite extends PlanTest with
SparkConnectPlanTest {
new java.util.ArrayList[Row](),
StructType(Seq(StructField("id", IntegerType), StructField("name",
StringType))))
+ lazy val sparkTestRelation3: DataFrame =
+ spark.createDataFrame(
+ new java.util.ArrayList[Row](),
+ StructType(Seq(StructField("id", IntegerType), StructField("date",
TimestampType))))
+
lazy val sparkTestRelationMap: DataFrame =
spark.createDataFrame(
new java.util.ArrayList[Row](),
@@ -93,6 +103,12 @@ class SparkConnectProtoSuite extends PlanTest with
SparkConnectPlanTest {
comparePlans(connectPlan, sparkPlan)
}
+ test("Basic select timestamp") {
+ val connectPlan = connectTestRelation3.select("date".protoAttr)
+ val sparkPlan = sparkTestRelation3.select("date")
+ comparePlans(connectPlan, sparkPlan)
+ }
+
test("Test select expression in strings") {
val connectPlan = connectTestRelation.selectExpr("abs(id)", "name")
val sparkPlan = sparkTestRelation.selectExpr("abs(id)", "name")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
