Repository: spark Updated Branches: refs/heads/master 23369c3bd -> cad29a40b
[SPARK-13728][SQL] Fix ORC PPD test so that pushed filters can be checked. ## What changes were proposed in this pull request? https://issues.apache.org/jira/browse/SPARK-13728 https://github.com/apache/spark/pull/11509 makes the output only single ORC file. It was 10 files but this PR writes only single file. So, this could not skip stripes in ORC by the pushed down filters. So, this PR simply repartitions data into 10 so that the test could pass. ## How was this patch tested? unittest and `./dev/run_tests` for code style test. Author: hyukjinkwon <gurwls...@gmail.com> Closes #11593 from HyukjinKwon/SPARK-13728. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cad29a40 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cad29a40 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cad29a40 Branch: refs/heads/master Commit: cad29a40b24a8e89f2d906e263866546f8ab6071 Parents: 23369c3 Author: hyukjinkwon <gurwls...@gmail.com> Authored: Wed Mar 9 10:48:53 2016 -0800 Committer: Michael Armbrust <mich...@databricks.com> Committed: Wed Mar 9 10:48:53 2016 -0800 ---------------------------------------------------------------------- .../scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/cad29a40/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala index 3c05266..9ca07e9 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala @@ -348,7 +348,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest { } } - ignore("SPARK-10623 Enable ORC PPD") { + test("SPARK-10623 Enable ORC PPD") { withTempPath { dir => withSQLConf(SQLConf.ORC_FILTER_PUSHDOWN_ENABLED.key -> "true") { import testImplicits._ @@ -363,7 +363,9 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest { val nullValue: Option[String] = None (maybeInt, nullValue) } - createDataFrame(data).toDF("a", "b").write.orc(path) + // It needs to repartition data so that we can have several ORC files + // in order to skip stripes in ORC. + createDataFrame(data).toDF("a", "b").repartition(10).write.orc(path) val df = sqlContext.read.orc(path) def checkPredicate(pred: Column, answer: Seq[Row]): Unit = { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
