This is an automated email from the ASF dual-hosted git repository. jhyde pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 8213277dc3e4e1ea0ddb64f7df9b34bf73305e3d Author: jacky <[email protected]> AuthorDate: Thu May 27 18:29:01 2021 +0800 [CALCITE-4606] In Elasticsearch adapter, translate SEARCH RexCall to termsQuery (Jacky Yin) Translate SEARCH RexCall(In/NotIn) to termsQuery in ES. Close apache/calcite#2420 --- .../adapter/elasticsearch/PredicateAnalyzer.java | 112 ++++++++++++++++++++- .../adapter/elasticsearch/AggregationTest.java | 43 +++++++- 2 files changed, 153 insertions(+), 2 deletions(-) diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java index 3eea1c4..b8e5c45 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java @@ -29,24 +29,31 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.NlsString; +import org.apache.calcite.util.Sarg; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; +import com.google.common.collect.Range; +import java.util.ArrayList; import java.util.GregorianCalendar; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.boolQuery; import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.existsQuery; import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.rangeQuery; import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.regexpQuery; import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.termQuery; +import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.termsQuery; import static java.lang.String.format; +import static java.util.Objects.requireNonNull; /** * Query predicate analyzer. Uses visitor pattern to traverse existing expression @@ -179,6 +186,13 @@ class PredicateAnalyzer { default: return false; } + case INTERNAL: + switch (call.getKind()) { + case SEARCH: + return canBeTranslatedToTermsQuery(call); + default: + return false; + } case FUNCTION_ID: case FUNCTION_STAR: default: @@ -186,6 +200,36 @@ class PredicateAnalyzer { } } + /** + * There are three types of the Sarg included in SEARCH RexCall: + * 1) Sarg is points (In ('a', 'b', 'c' ...)). + * In this case the search call can be translated to terms Query + * 2) Sarg is complementedPoints (Not in ('a', 'b')). + * In this case the search call can be translated to MustNot terms Query + * 3) Sarg is real Range( > 1 and <= 10). + * In this case the search call should be translated to rang Query + * Currently only the 1) and 2) cases are supported. + * @param search SEARCH RexCall + * @return true if it isSearchWithPoints or isSearchWithComplementedPoints, other false + */ + static boolean canBeTranslatedToTermsQuery(RexCall search) { + return isSearchWithPoints(search) || isSearchWithComplementedPoints(search); + } + + @SuppressWarnings("BetaApi") + static boolean isSearchWithPoints(RexCall search) { + RexLiteral literal = (RexLiteral) search.getOperands().get(1); + final Sarg<?> sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg"); + return sarg.isPoints(); + } + + @SuppressWarnings("BetaApi") + static boolean isSearchWithComplementedPoints(RexCall search) { + RexLiteral literal = (RexLiteral) search.getOperands().get(1); + final Sarg<?> sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg"); + return sarg.isComplementedPoints(); + } + @Override public Expression visitCall(RexCall call) { SqlSyntax syntax = call.getOperator().getSyntax(); @@ -201,6 +245,8 @@ class PredicateAnalyzer { return postfix(call); case PREFIX: return prefix(call); + case INTERNAL: + return binary(call); case SPECIAL: switch (call.getKind()) { case CAST: @@ -350,6 +396,12 @@ class PredicateAnalyzer { return QueryExpression.create(pair.getKey()).gte(pair.getValue()); } return QueryExpression.create(pair.getKey()).lte(pair.getValue()); + case SEARCH: + if (isSearchWithComplementedPoints(call)) { + return QueryExpression.create(pair.getKey()).notIn(pair.getValue()); + } else { + return QueryExpression.create(pair.getKey()).in(pair.getValue()); + } default: break; } @@ -533,6 +585,10 @@ class PredicateAnalyzer { public abstract QueryExpression equals(LiteralExpression literal); + public abstract QueryExpression in(LiteralExpression literal); + + public abstract QueryExpression notIn(LiteralExpression literal); + public abstract QueryExpression notEquals(LiteralExpression literal); public abstract QueryExpression gt(LiteralExpression literal); @@ -677,6 +733,14 @@ class PredicateAnalyzer { @Override public QueryExpression isTrue() { throw new PredicateAnalyzerException("isTrue cannot be applied to a compound expression"); } + + @Override public QueryExpression in(LiteralExpression literal) { + throw new PredicateAnalyzerException("in cannot be applied to a compound expression"); + } + + @Override public QueryExpression notIn(LiteralExpression literal) { + throw new PredicateAnalyzerException("notIn cannot be applied to a compound expression"); + } } /** @@ -797,6 +861,18 @@ class PredicateAnalyzer { builder = termQuery(getFieldReference(), true); return this; } + + @Override public QueryExpression in(LiteralExpression literal) { + Iterable<?> iterable = (Iterable<?>) literal.value(); + builder = termsQuery(getFieldReference(), iterable); + return this; + } + + @Override public QueryExpression notIn(LiteralExpression literal) { + Iterable<?> iterable = (Iterable<?>) literal.value(); + builder = boolQuery().mustNot(termsQuery(getFieldReference(), iterable)); + return this; + } } @@ -896,7 +972,9 @@ class PredicateAnalyzer { Object value() { - if (isIntegral()) { + if (isSarg()) { + return sargValue(); + } else if (isIntegral()) { return longValue(); } else if (isFloatingPoint()) { return doubleValue(); @@ -925,6 +1003,10 @@ class PredicateAnalyzer { return SqlTypeName.CHAR_TYPES.contains(literal.getType().getSqlTypeName()); } + public boolean isSarg() { + return SqlTypeName.SARG.getName().equalsIgnoreCase(literal.getTypeName().getName()); + } + long longValue() { return ((Number) literal.getValue()).longValue(); } @@ -941,6 +1023,34 @@ class PredicateAnalyzer { return RexLiteral.stringValue(literal); } + @SuppressWarnings("BetaApi") + List<Object> sargValue() { + final Sarg sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg"); + final RelDataType type = literal.getType(); + List<Object> values = new ArrayList<>(); + final SqlTypeName sqlTypeName = type.getSqlTypeName(); + if (sarg.isPoints()) { + Set<Range> ranges = sarg.rangeSet.asRanges(); + ranges.forEach(range -> + values.add(sargPointValue(range.lowerEndpoint(), sqlTypeName))); + } else if (sarg.isComplementedPoints()) { + Set<Range> ranges = sarg.negate().rangeSet.asRanges(); + ranges.forEach(range -> + values.add(sargPointValue(range.lowerEndpoint(), sqlTypeName))); + } + return values; + } + + Object sargPointValue(Object point, SqlTypeName sqlTypeName) { + switch (sqlTypeName) { + case CHAR: + case VARCHAR: + return ((NlsString) point).getValue(); + default: + return point; + } + } + Object rawValue() { return literal.getValue(); } diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java index f79edc8..385ad26 100644 --- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java +++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java @@ -47,7 +47,6 @@ import java.util.Map; /** * Testing Elasticsearch aggregation transformations. */ -@Disabled("RestClient often timeout in PR CI") @ResourceLock(value = "elasticsearch-scrolls", mode = ResourceAccessMode.READ) class AggregationTest { @@ -112,6 +111,33 @@ class AggregationTest { }; } + /** + * Currently the patterns like below will be converted to Search in range + * which is not supported in elastic search adapter. + * (val1 >= 10 and val1 <= 20) + * (val1 <= 10 or val1 >=20) + * (val1 <= 10) or (val1 > 15 and val1 <= 20) + * So disable this test case until the translation from Search in range + * to rang Query in ES is implemented. + */ + @Disabled + @Test void searchInRange() { + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*) from view where val1 >= 10 and val1 <=20") + .returns("EXPR$0=1\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*) from view where val1 <= 10 or val1 >=20") + .returns("EXPR$0=2\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*) from view where val1 <= 10 or (val1 > 15 and val1 <= 20)") + .returns("EXPR$0=2\n"); + } + @Test void countStar() { CalciteAssert.that() .with(newConnectionFactory()) @@ -130,6 +156,21 @@ class AggregationTest { .with(newConnectionFactory()) .query("select count(*) from view where cat1 in ('a', 'b')") .returns("EXPR$0=2\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*) from view where val1 in (10, 20)") + .returns("EXPR$0=0\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*) from view where cat4 in ('2018-01-01', '2019-12-12')") + .returns("EXPR$0=2\n"); + + CalciteAssert.that() + .with(newConnectionFactory()) + .query("select count(*) from view where cat4 not in ('2018-01-01', '2019-12-12')") + .returns("EXPR$0=1\n"); } @Test void all() {
