Repository: incubator-carbondata Updated Branches: refs/heads/master 25115c3f8 -> 6549b3680
Or filters are not working in spark 1.6. Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/cd2edc3d Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/cd2edc3d Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/cd2edc3d Branch: refs/heads/master Commit: cd2edc3df44eaa464a1fac002893943b436c2c3f Parents: 25115c3 Author: ravipesala <ravi.pes...@gmail.com> Authored: Thu Mar 23 22:17:48 2017 +0530 Committer: chenliang613 <chenliang...@huawei.com> Committed: Fri Mar 24 10:48:50 2017 +0530 ---------------------------------------------------------------------- .../apache/carbondata/spark/CarbonFilters.scala | 91 +++++--------------- 1 file changed, 20 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd2edc3d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala index 7a3c737..8063ccb 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala @@ -207,22 +207,34 @@ object CarbonFilters { carbonTable: CarbonTable): Option[CarbonExpression] = { def transformExpression(expr: Expression, or: Boolean = false): Option[CarbonExpression] = { expr match { - case or@ Or(left, right) => - val leftFilter = transformExpression(left, true) - val rightFilter = transformExpression(right, true) + case orFilter@ Or(left, right) => + val leftFilter = transformExpression(left, or = true) + val rightFilter = transformExpression(right, or = true) if (leftFilter.isDefined && rightFilter.isDefined) { Some(new OrExpression(leftFilter.get, rightFilter.get)) } else { - or.collect { - case attr: AttributeReference => attributesNeedToDecode.add(attr) + if (!or) { + orFilter.collect { + case attr: AttributeReference => attributesNeedToDecode.add(attr) + } + unprocessedExprs += orFilter } - unprocessedExprs += or None } case And(left, right) => - (transformExpression(left) ++ transformExpression(right)).reduceOption(new - AndExpression(_, _)) + val leftFilter = transformExpression(left, or) + val rightFilter = transformExpression(right, or) + if (or) { + if (leftFilter.isDefined && rightFilter.isDefined) { + (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _)) + } else { + None + } + } else { + (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _)) + } + case EqualTo(a: Attribute, l@Literal(v, t)) => Some( @@ -238,20 +250,6 @@ object CarbonFilters { transformExpression(l).get ) ) - case EqualTo(Cast(a: Attribute, _), l@Literal(v, t)) => - Some( - new EqualToExpression( - transformExpression(a).get, - transformExpression(l).get - ) - ) - case EqualTo(l@Literal(v, t), Cast(a: Attribute, _)) => - Some( - new EqualToExpression( - transformExpression(a).get, - transformExpression(l).get - ) - ) case Not(EqualTo(a: Attribute, l@Literal(v, t))) => Some( @@ -267,20 +265,6 @@ object CarbonFilters { transformExpression(l).get ) ) - case Not(EqualTo(Cast(a: Attribute, _), l@Literal(v, t))) => - Some( - new NotEqualsExpression( - transformExpression(a).get, - transformExpression(l).get - ) - ) - case Not(EqualTo(l@Literal(v, t), Cast(a: Attribute, _))) => - Some( - new NotEqualsExpression( - transformExpression(a).get, - transformExpression(l).get - ) - ) case IsNotNull(child: Attribute) => Some(new NotEqualsExpression(transformExpression(child).get, transformExpression(Literal(null)).get, true)) @@ -298,64 +282,30 @@ object CarbonFilters { case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) => Some(new InExpression(transformExpression(a).get, new ListExpression(convertToJavaList(list.map(transformExpression(_).get))))) - case Not(In(Cast(a: Attribute, _), list)) - if !list.exists(!_.isInstanceOf[Literal]) => - /* if any illogical expression comes in NOT IN Filter like - NOT IN('scala',NULL) this will be treated as false expression and will - always return no result. */ - if (list.exists(x => isNullLiteral(x.asInstanceOf[Literal]))) { - Some(new FalseExpression(transformExpression(a).get)) - } else { - Some(new NotInExpression(transformExpression(a).get, new ListExpression( - convertToJavaList(list.map(transformExpression(_).get))))) - } - case In(Cast(a: Attribute, _), list) if !list.exists(!_.isInstanceOf[Literal]) => - Some(new InExpression(transformExpression(a).get, - new ListExpression(convertToJavaList(list.map(transformExpression(_).get))))) case GreaterThan(a: Attribute, l@Literal(v, t)) => Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get)) - case GreaterThan(Cast(a: Attribute, _), l@Literal(v, t)) => - Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get)) case GreaterThan(l@Literal(v, t), a: Attribute) => Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get)) - case GreaterThan(l@Literal(v, t), Cast(a: Attribute, _)) => - Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get)) case LessThan(a: Attribute, l@Literal(v, t)) => Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get)) - case LessThan(Cast(a: Attribute, _), l@Literal(v, t)) => - Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get)) case LessThan(l@Literal(v, t), a: Attribute) => Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get)) - case LessThan(l@Literal(v, t), Cast(a: Attribute, _)) => - Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get)) case GreaterThanOrEqual(a: Attribute, l@Literal(v, t)) => Some(new GreaterThanEqualToExpression(transformExpression(a).get, transformExpression(l).get)) - case GreaterThanOrEqual(Cast(a: Attribute, _), l@Literal(v, t)) => - Some(new GreaterThanEqualToExpression(transformExpression(a).get, - transformExpression(l).get)) case GreaterThanOrEqual(l@Literal(v, t), a: Attribute) => Some(new LessThanEqualToExpression(transformExpression(a).get, transformExpression(l).get)) - case GreaterThanOrEqual(l@Literal(v, t), Cast(a: Attribute, _)) => - Some(new LessThanEqualToExpression(transformExpression(a).get, - transformExpression(l).get)) case LessThanOrEqual(a: Attribute, l@Literal(v, t)) => Some(new LessThanEqualToExpression(transformExpression(a).get, transformExpression(l).get)) - case LessThanOrEqual(Cast(a: Attribute, _), l@Literal(v, t)) => - Some(new LessThanEqualToExpression(transformExpression(a).get, - transformExpression(l).get)) case LessThanOrEqual(l@Literal(v, t), a: Attribute) => Some(new GreaterThanEqualToExpression(transformExpression(a).get, transformExpression(l).get)) - case LessThanOrEqual(l@Literal(v, t), Cast(a: Attribute, _)) => - Some(new GreaterThanEqualToExpression(transformExpression(a).get, - transformExpression(l).get)) case AttributeReference(name, dataType, _, _) => Some(new CarbonColumnExpression(name, @@ -363,7 +313,6 @@ object CarbonFilters { getActualCarbonDataType(name, carbonTable)))) case Literal(name, dataType) => Some(new CarbonLiteralExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(dataType))) - case Cast(left, right) if !left.isInstanceOf[Literal] => transformExpression(left) case others => if (!or) { others.collect {
