spark git commit: [SPARK-10623] [SQL] Fixes ORC predicate push-down
Repository: spark Updated Branches: refs/heads/master c8149ef2c -> 22be2ae14 [SPARK-10623] [SQL] Fixes ORC predicate push-down When pushing down a leaf predicate, ORC `SearchArgument` builder requires an extra "parent" predicate (any one among `AND`/`OR`/`NOT`) to wrap the leaf predicate. E.g., to push down `a < 1`, we must build `AND(a < 1)` instead. Fortunately, when actually constructing the `SearchArgument`, the builder will eliminate all those unnecessary wrappers. This PR is based on #8783 authored by zhzhan. I also took the chance to simply `OrcFilters` a little bit to improve readability. Author: Cheng Lian Closes #8799 from liancheng/spark-10623/fix-orc-ppd. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/22be2ae1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/22be2ae1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/22be2ae1 Branch: refs/heads/master Commit: 22be2ae147a111e88896f6fb42ed46bbf108a99b Parents: c8149ef Author: Cheng Lian Authored: Fri Sep 18 18:42:20 2015 -0700 Committer: Yin Huai Committed: Fri Sep 18 18:42:20 2015 -0700 -- .../apache/spark/sql/hive/orc/OrcFilters.scala | 56 .../spark/sql/hive/orc/OrcQuerySuite.scala | 30 +++ 2 files changed, 52 insertions(+), 34 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/22be2ae1/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala -- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala index b3d9f7f..27193f5 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala @@ -31,11 +31,13 @@ import org.apache.spark.sql.sources._ * and cannot be used anymore. */ private[orc] object OrcFilters extends Logging { - def createFilter(expr: Array[Filter]): Option[SearchArgument] = { -expr.reduceOption(And).flatMap { conjunction => - val builder = SearchArgumentFactory.newBuilder() - buildSearchArgument(conjunction, builder).map(_.build()) -} + def createFilter(filters: Array[Filter]): Option[SearchArgument] = { +for { + // Combines all filters with `And`s to produce a single conjunction predicate + conjunction <- filters.reduceOption(And) + // Then tries to build a single ORC `SearchArgument` for the conjunction predicate + builder <- buildSearchArgument(conjunction, SearchArgumentFactory.newBuilder()) +} yield builder.build() } private def buildSearchArgument(expression: Filter, builder: Builder): Option[Builder] = { @@ -102,46 +104,32 @@ private[orc] object OrcFilters extends Logging { negate <- buildSearchArgument(child, builder.startNot()) } yield negate.end() - case EqualTo(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.equals(attribute, _)) + case EqualTo(attribute, value) if isSearchableLiteral(value) => +Some(builder.startAnd().equals(attribute, value).end()) - case EqualNullSafe(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.nullSafeEquals(attribute, _)) + case EqualNullSafe(attribute, value) if isSearchableLiteral(value) => +Some(builder.startAnd().nullSafeEquals(attribute, value).end()) - case LessThan(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.lessThan(attribute, _)) + case LessThan(attribute, value) if isSearchableLiteral(value) => +Some(builder.startAnd().lessThan(attribute, value).end()) - case LessThanOrEqual(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.lessThanEquals(attribute, _)) + case LessThanOrEqual(attribute, value) if isSearchableLiteral(value) => +Some(builder.startAnd().lessThanEquals(attribute, value).end()) - case GreaterThan(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.startNot().lessThanEquals(attribute, _).end()) + case GreaterThan(attribute, value) if isSearchableLiteral(value) => +Some(builder.startNot().lessThanEquals(attribute, value).end()) - case GreaterThanOrEqual(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.startNot().lessThan(attribute, _).end()) + case GreaterThanOrEqual(attribute, value) if isSearchableLiteral(value) => +Some(builder.startNot().lessThan(attrib
spark git commit: [SPARK-10623] [SQL] Fixes ORC predicate push-down
Repository: spark Updated Branches: refs/heads/branch-1.5 a6c315358 -> b3f1e6533 [SPARK-10623] [SQL] Fixes ORC predicate push-down When pushing down a leaf predicate, ORC `SearchArgument` builder requires an extra "parent" predicate (any one among `AND`/`OR`/`NOT`) to wrap the leaf predicate. E.g., to push down `a < 1`, we must build `AND(a < 1)` instead. Fortunately, when actually constructing the `SearchArgument`, the builder will eliminate all those unnecessary wrappers. This PR is based on #8783 authored by zhzhan. I also took the chance to simply `OrcFilters` a little bit to improve readability. Author: Cheng Lian Closes #8799 from liancheng/spark-10623/fix-orc-ppd. (cherry picked from commit 22be2ae147a111e88896f6fb42ed46bbf108a99b) Signed-off-by: Yin Huai Conflicts: sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b3f1e653 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b3f1e653 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b3f1e653 Branch: refs/heads/branch-1.5 Commit: b3f1e653320e074fe78971a2a3b659c36da20b45 Parents: a6c3153 Author: Cheng Lian Authored: Fri Sep 18 18:42:20 2015 -0700 Committer: Yin Huai Committed: Fri Sep 18 18:46:53 2015 -0700 -- .../apache/spark/sql/hive/orc/OrcFilters.scala | 53 +--- .../spark/sql/hive/orc/OrcQuerySuite.scala | 30 +++ 2 files changed, 53 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b3f1e653/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala -- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala index 86142e5..27193f5 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala @@ -31,11 +31,13 @@ import org.apache.spark.sql.sources._ * and cannot be used anymore. */ private[orc] object OrcFilters extends Logging { - def createFilter(expr: Array[Filter]): Option[SearchArgument] = { -expr.reduceOption(And).flatMap { conjunction => - val builder = SearchArgumentFactory.newBuilder() - buildSearchArgument(conjunction, builder).map(_.build()) -} + def createFilter(filters: Array[Filter]): Option[SearchArgument] = { +for { + // Combines all filters with `And`s to produce a single conjunction predicate + conjunction <- filters.reduceOption(And) + // Then tries to build a single ORC `SearchArgument` for the conjunction predicate + builder <- buildSearchArgument(conjunction, SearchArgumentFactory.newBuilder()) +} yield builder.build() } private def buildSearchArgument(expression: Filter, builder: Builder): Option[Builder] = { @@ -102,41 +104,32 @@ private[orc] object OrcFilters extends Logging { negate <- buildSearchArgument(child, builder.startNot()) } yield negate.end() - case EqualTo(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.equals(attribute, _)) + case EqualTo(attribute, value) if isSearchableLiteral(value) => +Some(builder.startAnd().equals(attribute, value).end()) + + case EqualNullSafe(attribute, value) if isSearchableLiteral(value) => +Some(builder.startAnd().nullSafeEquals(attribute, value).end()) - case LessThan(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.lessThan(attribute, _)) + case LessThan(attribute, value) if isSearchableLiteral(value) => +Some(builder.startAnd().lessThan(attribute, value).end()) - case LessThanOrEqual(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.lessThanEquals(attribute, _)) + case LessThanOrEqual(attribute, value) if isSearchableLiteral(value) => +Some(builder.startAnd().lessThanEquals(attribute, value).end()) - case GreaterThan(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.startNot().lessThanEquals(attribute, _).end()) + case GreaterThan(attribute, value) if isSearchableLiteral(value) => +Some(builder.startNot().lessThanEquals(attribute, value).end()) - case GreaterThanOrEqual(attribute, value) => -Option(value) - .filter(isSearchableLiteral) - .map(builder.startNot().lessThan(attribute, _).end()) + case GreaterThanOrEqual(attribute, value) if isSearchableLiteral(value) => +Some(buil