Repository: incubator-carbondata Updated Branches: refs/heads/master e5ed64bbe -> d3d2e338d
Pushdown greaterthan and lessthan filters to carbon Rebased code Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/175a21fb Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/175a21fb Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/175a21fb Branch: refs/heads/master Commit: 175a21fb37e1dc9fafcbdc28a299eae9e86c056c Parents: e5ed64b Author: ravipesala <ravi.pes...@gmail.com> Authored: Thu Jul 14 00:56:59 2016 +0530 Committer: jackylk <jacky.li...@huawei.com> Committed: Tue Jul 19 22:11:20 2016 +0800 ---------------------------------------------------------------------- .../org/carbondata/spark/CarbonFilters.scala | 97 +++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/175a21fb/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala b/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala index a7a8313..fd0ccaf 100644 --- a/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala +++ b/integration/spark/src/main/scala/org/carbondata/spark/CarbonFilters.scala @@ -34,12 +34,12 @@ import org.carbondata.scan.expression.logical.{AndExpression, OrExpression} import org.carbondata.spark.util.CarbonScalaUtil /** - * All filter coversions are done here. + * All filter conversions are done here. */ object CarbonFilters { /** - * Converts data sources filters to Parquet filter predicates. + * Converts data sources filters to carbon filter predicates. */ def createCarbonFilter(schema: StructType, predicate: sources.Filter): Option[CarbonExpression] = { @@ -62,6 +62,19 @@ object CarbonFilters { Some(new NotEqualsExpression(getCarbonExpression(name), getCarbonLiteralExpression(name, value))) + case sources.GreaterThan(name, value) => + Some(new GreaterThanExpression(getCarbonExpression(name), + getCarbonLiteralExpression(name, value))) + case sources.LessThan(name, value) => + Some(new LessThanExpression(getCarbonExpression(name), + getCarbonLiteralExpression(name, value))) + case sources.GreaterThanOrEqual(name, value) => + Some(new GreaterThanEqualToExpression(getCarbonExpression(name), + getCarbonLiteralExpression(name, value))) + case sources.LessThanOrEqual(name, value) => + Some(new LessThanEqualToExpression(getCarbonExpression(name), + getCarbonLiteralExpression(name, value))) + case sources.In(name, values) => Some(new InExpression(getCarbonExpression(name), new ListExpression(values.map(f => getCarbonLiteralExpression(name, f)).toList.asJava))) @@ -153,6 +166,42 @@ object CarbonFilters { val hSet = list.map(e => e.eval(EmptyRow)) Some(sources.In(a.name, hSet.toArray)) + case GreaterThan(a: Attribute, Literal(v, t)) => + Some(sources.GreaterThan(a.name, v)) + case GreaterThan(Literal(v, t), a: Attribute) => + Some(sources.LessThan(a.name, v)) + case GreaterThan(Cast(a: Attribute, _), Literal(v, t)) => + Some(sources.GreaterThan(a.name, v)) + case GreaterThan(Literal(v, t), Cast(a: Attribute, _)) => + Some(sources.LessThan(a.name, v)) + + case LessThan(a: Attribute, Literal(v, t)) => + Some(sources.LessThan(a.name, v)) + case LessThan(Literal(v, t), a: Attribute) => + Some(sources.GreaterThan(a.name, v)) + case LessThan(Cast(a: Attribute, _), Literal(v, t)) => + Some(sources.LessThan(a.name, v)) + case LessThan(Literal(v, t), Cast(a: Attribute, _)) => + Some(sources.GreaterThan(a.name, v)) + + case GreaterThanOrEqual(a: Attribute, Literal(v, t)) => + Some(sources.GreaterThanOrEqual(a.name, v)) + case GreaterThanOrEqual(Literal(v, t), a: Attribute) => + Some(sources.LessThanOrEqual(a.name, v)) + case GreaterThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) => + Some(sources.GreaterThanOrEqual(a.name, v)) + case GreaterThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) => + Some(sources.LessThanOrEqual(a.name, v)) + + case LessThanOrEqual(a: Attribute, Literal(v, t)) => + Some(sources.LessThanOrEqual(a.name, v)) + case LessThanOrEqual(Literal(v, t), a: Attribute) => + Some(sources.GreaterThanOrEqual(a.name, v)) + case LessThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) => + Some(sources.LessThanOrEqual(a.name, v)) + case LessThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) => + Some(sources.GreaterThanOrEqual(a.name, v)) + case others => if (!or) { others.collect { @@ -221,6 +270,50 @@ object CarbonFilters { Some(new InExpression(transformExpression(a).get, new ListExpression(list.map(transformExpression(_).get).asJava))) + case GreaterThan(a: Attribute, l@Literal(v, t)) => + Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get)) + case GreaterThan(Cast(a: Attribute, _), l@Literal(v, t)) => + Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get)) + case GreaterThan(l@Literal(v, t), a: Attribute) => + Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get)) + case GreaterThan(l@Literal(v, t), Cast(a: Attribute, _)) => + Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get)) + + case LessThan(a: Attribute, l@Literal(v, t)) => + Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get)) + case LessThan(Cast(a: Attribute, _), l@Literal(v, t)) => + Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get)) + case LessThan(l@Literal(v, t), a: Attribute) => + Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get)) + case LessThan(l@Literal(v, t), Cast(a: Attribute, _)) => + Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get)) + + case GreaterThanOrEqual(a: Attribute, l@Literal(v, t)) => + Some(new GreaterThanEqualToExpression(transformExpression(a).get, + transformExpression(l).get)) + case GreaterThanOrEqual(Cast(a: Attribute, _), l@Literal(v, t)) => + Some(new GreaterThanEqualToExpression(transformExpression(a).get, + transformExpression(l).get)) + case GreaterThanOrEqual(l@Literal(v, t), a: Attribute) => + Some(new LessThanEqualToExpression(transformExpression(a).get, + transformExpression(l).get)) + case GreaterThanOrEqual(l@Literal(v, t), Cast(a: Attribute, _)) => + Some(new LessThanEqualToExpression(transformExpression(a).get, + transformExpression(l).get)) + + case LessThanOrEqual(a: Attribute, l@Literal(v, t)) => + Some(new LessThanEqualToExpression(transformExpression(a).get, + transformExpression(l).get)) + case LessThanOrEqual(Cast(a: Attribute, _), l@Literal(v, t)) => + Some(new LessThanEqualToExpression(transformExpression(a).get, + transformExpression(l).get)) + case LessThanOrEqual(l@Literal(v, t), a: Attribute) => + Some(new GreaterThanEqualToExpression(transformExpression(a).get, + transformExpression(l).get)) + case LessThanOrEqual(l@Literal(v, t), Cast(a: Attribute, _)) => + Some(new GreaterThanEqualToExpression(transformExpression(a).get, + transformExpression(l).get)) + case AttributeReference(name, dataType, _, _) => Some(new CarbonColumnExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(
