spark git commit: [SPARK-10477][SQL] using DSL in ColumnPruningSuite to improve readability
Repository: spark Updated Branches: refs/heads/branch-1.6 93095eb29 -> fb08f7b78 [SPARK-10477][SQL] using DSL in ColumnPruningSuite to improve readability Author: Wenchen Fan Closes #8645 from cloud-fan/test. (cherry picked from commit a89e8b6122ee5a1517fbcf405b1686619db56696) Signed-off-by: Andrew Or Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fb08f7b7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fb08f7b7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fb08f7b7 Branch: refs/heads/branch-1.6 Commit: fb08f7b784bc8b5e0cd110f315f72c7d9fc65e08 Parents: 93095eb Author: Wenchen Fan Authored: Tue Dec 15 18:29:19 2015 -0800 Committer: Andrew Or Committed: Tue Dec 15 18:29:25 2015 -0800 -- .../apache/spark/sql/catalyst/dsl/package.scala | 7 ++-- .../catalyst/optimizer/ColumnPruningSuite.scala | 41 +++- 2 files changed, 27 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/fb08f7b7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala index af594c2..e509711 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala @@ -275,13 +275,14 @@ package object dsl { def unionAll(otherPlan: LogicalPlan): LogicalPlan = Union(logicalPlan, otherPlan) - // TODO specify the output column names def generate( generator: Generator, join: Boolean = false, outer: Boolean = false, -alias: Option[String] = None): LogicalPlan = -Generate(generator, join = join, outer = outer, alias, Nil, logicalPlan) +alias: Option[String] = None, +outputNames: Seq[String] = Nil): LogicalPlan = +Generate(generator, join = join, outer = outer, alias, + outputNames.map(UnresolvedAttribute(_)), logicalPlan) def insertInto(tableName: String, overwrite: Boolean = false): LogicalPlan = InsertIntoTable( http://git-wip-us.apache.org/repos/asf/spark/blob/fb08f7b7/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala index 4a1e7ce..9bf61ae 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer import org.apache.spark.sql.catalyst.expressions.Explode import org.apache.spark.sql.catalyst.plans.PlanTest -import org.apache.spark.sql.catalyst.plans.logical.{Project, LocalRelation, Generate, LogicalPlan} +import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan} import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ @@ -35,12 +35,11 @@ class ColumnPruningSuite extends PlanTest { test("Column pruning for Generate when Generate.join = false") { val input = LocalRelation('a.int, 'b.array(StringType)) -val query = Generate(Explode('b), false, false, None, 's.string :: Nil, input).analyze +val query = input.generate(Explode('b), join = false).analyze + val optimized = Optimize.execute(query) -val correctAnswer = - Generate(Explode('b), false, false, None, 's.string :: Nil, -Project('b.attr :: Nil, input)).analyze +val correctAnswer = input.select('b).generate(Explode('b), join = false).analyze comparePlans(optimized, correctAnswer) } @@ -49,16 +48,19 @@ class ColumnPruningSuite extends PlanTest { val input = LocalRelation('a.int, 'b.int, 'c.array(StringType)) val query = - Project(Seq('a, 's), -Generate(Explode('c), true, false, None, 's.string :: Nil, - input)).analyze + input +.generate(Explode('c), join = true, outputNames = "explode" :: Nil) +.select('a, 'explode) +.analyze + val optimized = Optimize.execute(query) val correctAnswer = - Project(Seq('a, 's), -Generate(Explode('c), true, false, None, 's.string :: Nil, - Project(Seq('a, 'c), -input))).analyze + input +
spark git commit: [SPARK-10477][SQL] using DSL in ColumnPruningSuite to improve readability
Repository: spark Updated Branches: refs/heads/master c5b6b398d -> a89e8b612 [SPARK-10477][SQL] using DSL in ColumnPruningSuite to improve readability Author: Wenchen Fan Closes #8645 from cloud-fan/test. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a89e8b61 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a89e8b61 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a89e8b61 Branch: refs/heads/master Commit: a89e8b6122ee5a1517fbcf405b1686619db56696 Parents: c5b6b39 Author: Wenchen Fan Authored: Tue Dec 15 18:29:19 2015 -0800 Committer: Andrew Or Committed: Tue Dec 15 18:29:19 2015 -0800 -- .../apache/spark/sql/catalyst/dsl/package.scala | 7 ++-- .../catalyst/optimizer/ColumnPruningSuite.scala | 41 +++- 2 files changed, 27 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/a89e8b61/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala index af594c2..e509711 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala @@ -275,13 +275,14 @@ package object dsl { def unionAll(otherPlan: LogicalPlan): LogicalPlan = Union(logicalPlan, otherPlan) - // TODO specify the output column names def generate( generator: Generator, join: Boolean = false, outer: Boolean = false, -alias: Option[String] = None): LogicalPlan = -Generate(generator, join = join, outer = outer, alias, Nil, logicalPlan) +alias: Option[String] = None, +outputNames: Seq[String] = Nil): LogicalPlan = +Generate(generator, join = join, outer = outer, alias, + outputNames.map(UnresolvedAttribute(_)), logicalPlan) def insertInto(tableName: String, overwrite: Boolean = false): LogicalPlan = InsertIntoTable( http://git-wip-us.apache.org/repos/asf/spark/blob/a89e8b61/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala -- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala index 4a1e7ce..9bf61ae 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer import org.apache.spark.sql.catalyst.expressions.Explode import org.apache.spark.sql.catalyst.plans.PlanTest -import org.apache.spark.sql.catalyst.plans.logical.{Project, LocalRelation, Generate, LogicalPlan} +import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan} import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ @@ -35,12 +35,11 @@ class ColumnPruningSuite extends PlanTest { test("Column pruning for Generate when Generate.join = false") { val input = LocalRelation('a.int, 'b.array(StringType)) -val query = Generate(Explode('b), false, false, None, 's.string :: Nil, input).analyze +val query = input.generate(Explode('b), join = false).analyze + val optimized = Optimize.execute(query) -val correctAnswer = - Generate(Explode('b), false, false, None, 's.string :: Nil, -Project('b.attr :: Nil, input)).analyze +val correctAnswer = input.select('b).generate(Explode('b), join = false).analyze comparePlans(optimized, correctAnswer) } @@ -49,16 +48,19 @@ class ColumnPruningSuite extends PlanTest { val input = LocalRelation('a.int, 'b.int, 'c.array(StringType)) val query = - Project(Seq('a, 's), -Generate(Explode('c), true, false, None, 's.string :: Nil, - input)).analyze + input +.generate(Explode('c), join = true, outputNames = "explode" :: Nil) +.select('a, 'explode) +.analyze + val optimized = Optimize.execute(query) val correctAnswer = - Project(Seq('a, 's), -Generate(Explode('c), true, false, None, 's.string :: Nil, - Project(Seq('a, 'c), -input))).analyze + input +.select('a, 'c) +.generate(Explode('c), join = true, outputNames = "explode" :: Nil) +