[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user asfgit closed the pull request at: https://github.com/apache/spark/pull/13704 --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r76721258
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+comparePlans(optimized, plan)
+ }
+
+ test("nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to non-nullable map cast") {
+val input = LocalRelation('m.map(MapType(StringType, StringType,
false)))
+val plan = input.select('m.cast(MapType(StringType, StringType, false))
+ .as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('m.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable map cast") {
+val input = LocalRelation('m.map(MapType(StringType, StringType,
false)))
+val plan = input.select('m.cast(MapType(StringType, StringType, true))
+ .as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('m.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable map cast") {
+val input = LocalRelation('m.map(MapType(StringType, StringType,
true)))
+val plan = input.select('m.cast(MapType(StringType, StringType, false))
+ .as("casted")).analyze
+val optimized = Optimize.execute(plan)
+comparePlans(optimized, plan)
+ }
+
+ test("nullable to nullable map cast") {
+val input = LocalRelation('m.map(MapType(StringType, StringType,
true)))
+val plan = input.select('m.cast(MapType(StringType, StringType, true))
+ .as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('m.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+}
--- End diff --
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r76721180
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+comparePlans(optimized, plan)
+ }
+
+ test("nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to non-nullable map cast") {
+val input = LocalRelation('m.map(MapType(StringType, StringType,
false)))
+val plan = input.select('m.cast(MapType(StringType, StringType, false))
+ .as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('m.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable map cast") {
--- End diff --
non-nullable value map to nullable value map cast
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r76721154
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
--- End diff --
`non-nullable element array to nullable element array cast`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r73824598
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -1441,6 +1441,12 @@ object PushPredicateThroughJoin extends
Rule[LogicalPlan] with PredicateHelper {
object SimplifyCasts extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions
{
case Cast(e, dataType) if e.dataType == dataType => e
+case c @ Cast(e, dataType) => (e.dataType, dataType) match {
--- End diff --
@liancheng Could you take a look?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r72915873
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -1441,6 +1441,12 @@ object PushPredicateThroughJoin extends
Rule[LogicalPlan] with PredicateHelper {
object SimplifyCasts extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions
{
case Cast(e, dataType) if e.dataType == dataType => e
+case c @ Cast(e, dataType) => (e.dataType, dataType) match {
--- End diff --
ping @liancheng
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r72116259
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -1441,6 +1441,12 @@ object PushPredicateThroughJoin extends
Rule[LogicalPlan] with PredicateHelper {
object SimplifyCasts extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions
{
case Cast(e, dataType) if e.dataType == dataType => e
+case c @ Cast(e, dataType) => (e.dataType, dataType) match {
--- End diff --
These cases handle a cast only from non-nullable type. If I am correct, a
cast from a nullable type is handled as previously done. cc @cloud-fan
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r72080255
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -1441,6 +1441,12 @@ object PushPredicateThroughJoin extends
Rule[LogicalPlan] with PredicateHelper {
object SimplifyCasts extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions
{
case Cast(e, dataType) if e.dataType == dataType => e
+case c @ Cast(e, dataType) => (e.dataType, dataType) match {
--- End diff --
It would be nice if somehow we can narrow the scope of this optimization to
the exact case described in this PR. In general, making a nullable type
non-nullable is not safe.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r72077898
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -1441,6 +1441,12 @@ object PushPredicateThroughJoin extends
Rule[LogicalPlan] with PredicateHelper {
object SimplifyCasts extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions
{
case Cast(e, dataType) if e.dataType == dataType => e
+case c @ Cast(e, dataType) => (e.dataType, dataType) match {
--- End diff --
As long as we only do this kind of casting in the case mentioned above, it
should be safe. But currently I'm not sure whether that is true.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r71346126
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = 'a.array(ArrayType(IntegerType, false))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+comparePlans(optimized, plan)
+ }
+
+ test("nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ def map(keyType: DataType, valueType: DataType, nullable: Boolean):
AttributeReference =
--- End diff --
I eliminated them by using the similar approach to what we did for `Array`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r71333538
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = 'a.array(ArrayType(IntegerType, false))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+comparePlans(optimized, plan)
+ }
+
+ test("nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ def map(keyType: DataType, valueType: DataType, nullable: Boolean):
AttributeReference =
--- End diff --
but this
[map()](https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala#L249)
can, isn't it?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r71278812
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = 'a.array(ArrayType(IntegerType, false))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+comparePlans(optimized, plan)
+ }
+
+ test("nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ def map(keyType: DataType, valueType: DataType, nullable: Boolean):
AttributeReference =
--- End diff --
This is because [current
map()](https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala#L246)
cannot pass information on `nullable`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r71275696
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = 'a.array(ArrayType(IntegerType, false))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+comparePlans(optimized, plan)
+ }
+
+ test("nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val plan = input.select('a.cast(ArrayType(IntegerType,
true)).as("casted")).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select('a.as("casted")).analyze
+comparePlans(optimized, expected)
+ }
+
+ def map(keyType: DataType, valueType: DataType, nullable: Boolean):
AttributeReference =
--- End diff --
why need this?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r71275743
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -1441,6 +1441,12 @@ object PushPredicateThroughJoin extends
Rule[LogicalPlan] with PredicateHelper {
object SimplifyCasts extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions
{
case Cast(e, dataType) if e.dataType == dataType => e
+case c @ Cast(e, dataType) => (e.dataType, dataType) match {
--- End diff --
cc @yhuai @liancheng, is it always safe to do it?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r71260242
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = 'a.array(ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
--- End diff --
`val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted")).analyze`
you need to resolve the plan before optimize it
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r71215193
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = 'a.array(ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
--- End diff --
The following code caused an exception. Could you please let me know my
mistakes?
```java
test("non-nullable to non-nullable array cast") {
val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
val plan = input.select('a.cast(ArrayType(IntegerType,
false)).as("casted"))
val optimized = Optimize.execute(plan)
comparePlans(optimized, plan)
}
```
```java
Invalid call to dataType on unresolved object, tree: 'a
org.apache.spark.sql.catalyst.analysis.UnresolvedException: Invalid call to
dataType on unresolved object, tree: 'a
at
org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute.dataType(unresolved.scala:61)
at
org.apache.spark.sql.catalyst.optimizer.SimplifyCasts$$anonfun$apply$33.applyOrElse(Optimizer.scala:1443)
at
org.apache.spark.sql.catalyst.optimizer.SimplifyCasts$$anonfun$apply$33.applyOrElse(Optimizer.scala:1442)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:279)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:279)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:69)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:278)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:284)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:284)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.apply(TreeNode.scala:322)
at
org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:179)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformChildren(TreeNode.scala:320)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:284)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionDown$1(QueryPlan.scala:156)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$1(QueryPlan.scala:166)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$1$1.apply(QueryPlan.scala:170)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.AbstractTraversable.map(Traversable.scala:104)
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r71168135
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = 'a.array(ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
--- End diff --
can we just write `input.select('a.cast(ArrayType(IntegerType,
false)).as("casted"))`? then we don't need `array_intPrimitive`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r70562239
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ def array(arrayType: ArrayType): AttributeReference =
--- End diff --
can we put in into
https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala#L242?
Then in the test we can just write `val input =
LocalRelation('a.array(ArrayType(IntegerType, nullable = false)))`, and no need
to use `array_intPrimitive`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r70418445
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType)))
--- End diff --
for map, we have
```
def map(keyType: DataType, valueType: DataType): AttributeReference =
map(MapType(keyType, valueType))
def map(mapType: MapType): AttributeReference =
AttributeReference(s, mapType, nullable = true)()
```
we can also add similar one to array
```
def array(arrayType: ArrayType): AttributeReference
```
Then here we can create an array type attribute with expected nullability,
and avoid using literal in optimizer test
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r70404870
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, true)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val array_intNull = Literal.create(
+ Seq(1, 2, null, 4, 5), ArrayType(IntegerType, true))
+val plan = input.select(array_intNull
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+assert(optimized.resolved === false)
--- End diff --
I am very sorry. I made some mistakes before. Now, `comparePlans(optimized,
plan)` worked.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r70402730
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, true)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val array_intNull = Literal.create(
+ Seq(1, 2, null, 4, 5), ArrayType(IntegerType, true))
+val plan = input.select(array_intNull
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+assert(optimized.resolved === false)
--- End diff --
> This is because `plan.resolved == true` while `optimized.resolved ==
false`.
I'm confused about it, if the cast should produce "unresolved", why
`plan.resolved == true`?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r70398569
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, true)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val array_intNull = Literal.create(
+ Seq(1, 2, null, 4, 5), ArrayType(IntegerType, true))
+val plan = input.select(array_intNull
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+assert(optimized.resolved === false)
--- End diff --
I thought that `optimized.resolved = false` is an expected result. This
because a cast from `ArrayType(IntegerType, true)` to `ArrayType(IntegerType,
false)` produces `resolved == false` without an optimization in [this test
case](https://github.com/apache/spark/blob/master/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala#L521).
I would appreciate it if you correct my understanding.
In addition, I will check my test case.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r70397586
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, true)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val array_intNull = Literal.create(
+ Seq(1, 2, null, 4, 5), ArrayType(IntegerType, true))
+val plan = input.select(array_intNull
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+assert(optimized.resolved === false)
--- End diff --
how could this be? If one optimizer rule makes a resolved plan to
unresolved, then these must be something wrong in that rule.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r70395547
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, true)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val array_intNull = Literal.create(
+ Seq(1, 2, null, 4, 5), ArrayType(IntegerType, true))
+val plan = input.select(array_intNull
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+assert(optimized.resolved === false)
--- End diff --
When I just added 'comparePlan(optimized, plan') after this assert, this
comparison failed. This is because `plan.resolved == true` while
`optimized.resolved == false`.
Should we create an `expected` plan whose plan is similar to `plan` with
`resolved = false`?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13704: [SPARK-15985][SQL] Eliminate redundant cast from ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/13704#discussion_r70386711
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
---
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types._
+
+class SimplifyCastsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+val batches = Batch("SimplifyCasts", FixedPoint(50), SimplifyCasts) ::
Nil
+ }
+
+ test("non-nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("non-nullable to nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, false)))
+val array_intPrimitive = Literal.create(
+ Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, false))
+val plan = input.select(array_intPrimitive
+ .cast(ArrayType(IntegerType, true)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+val expected = input.select(array_intPrimitive.as('a)).analyze
+comparePlans(optimized, expected)
+ }
+
+ test("nullable to non-nullable array cast") {
+val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
+val array_intNull = Literal.create(
+ Seq(1, 2, null, 4, 5), ArrayType(IntegerType, true))
+val plan = input.select(array_intNull
+ .cast(ArrayType(IntegerType, false)).as('a)).analyze
+val optimized = Optimize.execute(plan)
+assert(optimized.resolved === false)
--- End diff --
we can check `comparePlans(optimized, plan)`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
