This is an automated email from the ASF dual-hosted git repository. caicancai pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 56668af6bfc9e0537dcb6389ecd78727185d1767 Author: Cancai Cai <[email protected]> AuthorDate: Fri Jun 19 15:32:39 2026 +0800 Revert "[CALCITE-7580] Remove Gandiva dependency from Arrow adapter" This reverts commit bf2b81b27d76ef3571ba676dbc7bce4ccb4a0413. --- arrow/build.gradle.kts | 1 + .../adapter/arrow/AbstractArrowEnumerator.java | 30 +-- .../adapter/arrow/ArrowDirectEnumerator.java | 31 ++- .../calcite/adapter/arrow/ArrowEnumerable.java | 25 ++- .../adapter/arrow/ArrowFilterEnumerator.java | 238 +++++---------------- .../adapter/arrow/ArrowProjectEnumerator.java | 80 +++++++ .../apache/calcite/adapter/arrow/ArrowRules.java | 2 +- .../apache/calcite/adapter/arrow/ArrowTable.java | 143 ++++++++++++- .../calcite/adapter/arrow/ArrowTranslator.java | 69 ++---- .../calcite/adapter/arrow/ConditionToken.java | 51 +---- .../calcite/adapter/arrow/ArrowAdapterTest.java | 50 ----- .../calcite/adapter/arrow/ArrowDataTest.java | 29 --- .../calcite/adapter/arrow/ArrowExtension.java | 23 +- bom/build.gradle.kts | 1 + .../java/org/apache/calcite/rex/RexSimplify.java | 26 ++- .../calcite/sql/type/SqlTypeFactoryImpl.java | 2 +- .../apache/calcite/sql2rel/SqlToRelConverter.java | 34 ++- .../org/apache/calcite/rex/RexProgramTest.java | 20 ++ .../calcite/sql/type/SqlTypeFactoryTest.java | 25 +++ .../apache/calcite/test/SqlToRelConverterTest.java | 18 ++ .../apache/calcite/test/SqlToRelConverterTest.xml | 24 +++ gradle.properties | 1 + site/_docs/history.md | 5 - .../org/apache/calcite/test/SqlToRelFixture.java | 5 + 24 files changed, 525 insertions(+), 408 deletions(-) diff --git a/arrow/build.gradle.kts b/arrow/build.gradle.kts index c75a8b5f67..598aa8a879 100644 --- a/arrow/build.gradle.kts +++ b/arrow/build.gradle.kts @@ -23,6 +23,7 @@ implementation("com.google.guava:guava") implementation("org.apache.arrow:arrow-memory-netty") implementation("org.apache.arrow:arrow-vector") + implementation("org.apache.arrow.gandiva:arrow-gandiva") annotationProcessor("org.immutables:value") compileOnly("org.immutables:value-annotations") diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java index 486e3f60bd..e188757b0d 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java @@ -24,7 +24,9 @@ import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -49,6 +51,8 @@ abstract class AbstractArrowEnumerator implements Enumerator<Object> { this.currRowIndex = -1; } + abstract void evaluateOperator(ArrowRecordBatch arrowRecordBatch); + protected void loadNextArrowBatch() { try { final VectorSchemaRoot vsr = arrowFileReader.getVectorSchemaRoot(); @@ -56,32 +60,14 @@ protected void loadNextArrowBatch() { this.valueVectors.add(vsr.getVector(i)); } this.rowCount = vsr.getRowCount(); + VectorUnloader vectorUnloader = new VectorUnloader(vsr); + ArrowRecordBatch arrowRecordBatch = vectorUnloader.getRecordBatch(); + evaluateOperator(arrowRecordBatch); } catch (IOException e) { throw Util.toUnchecked(e); } } - /** Loads the next non-empty Arrow batch. */ - protected boolean loadNextNonEmptyArrowBatch() { - while (true) { - final boolean hasNextBatch; - try { - hasNextBatch = arrowFileReader.loadNextBatch(); - } catch (IOException e) { - throw Util.toUnchecked(e); - } - if (!hasNextBatch) { - return false; - } - currRowIndex = -1; - valueVectors.clear(); - loadNextArrowBatch(); - if (rowCount > 0) { - return true; - } - } - } - @Override public Object current() { if (fields.size() == 1) { return getValue(this.valueVectors.get(0), currRowIndex); @@ -99,7 +85,7 @@ protected boolean loadNextNonEmptyArrowBatch() { * <p>For {@link TimeStampVector}, converts the raw value to * milliseconds since epoch, which is the representation used by * Calcite's Enumerable runtime for TIMESTAMP types. */ - protected static Object getValue(ValueVector vector, int index) { + private static Object getValue(ValueVector vector, int index) { if (vector instanceof TimeStampVector) { if (vector.isNull(index)) { return null; diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java index 787ffd88d9..0cdec7baeb 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java @@ -17,11 +17,19 @@ package org.apache.calcite.adapter.arrow; import org.apache.calcite.util.ImmutableIntList; +import org.apache.calcite.util.Util; import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; + +import java.io.IOException; /** * Enumerator that reads projected Arrow value-vectors directly. + * + * <p>This path is used for identity projections that Gandiva cannot project + * through the existing {@code Projector} path, such as Arrow binary vectors. + * It is not a replacement for Gandiva expression evaluation. */ class ArrowDirectEnumerator extends AbstractArrowEnumerator { private final Runnable onClose; @@ -32,14 +40,27 @@ class ArrowDirectEnumerator extends AbstractArrowEnumerator { this.onClose = onClose; } + @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { + } + @Override public boolean moveNext() { - while (currRowIndex >= rowCount - 1) { - if (!loadNextNonEmptyArrowBatch()) { - return false; + if (currRowIndex >= rowCount - 1) { + final boolean hasNextBatch; + try { + hasNextBatch = arrowFileReader.loadNextBatch(); + } catch (IOException e) { + throw Util.toUnchecked(e); + } + if (hasNextBatch) { + currRowIndex = 0; + this.valueVectors.clear(); + loadNextArrowBatch(); } + return hasNextBatch; + } else { + currRowIndex++; + return true; } - currRowIndex++; - return true; } @Override public void close() { diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java index b9c0c4171e..84ed5997aa 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java @@ -21,10 +21,11 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; +import org.apache.arrow.gandiva.evaluator.Filter; +import org.apache.arrow.gandiva.evaluator.Projector; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.types.pojo.Schema; -import java.util.List; +import org.checkerframework.checker.nullness.qual.Nullable; /** * Enumerable that reads from Arrow value-vectors. @@ -32,24 +33,28 @@ class ArrowEnumerable extends AbstractEnumerable<Object> { private final ArrowFileReader arrowFileReader; private final ImmutableIntList fields; - private final List<List<List<String>>> conditions; - private final Schema schema; + private final @Nullable Projector projector; + private final @Nullable Filter filter; private final Runnable onClose; ArrowEnumerable(ArrowFileReader arrowFileReader, ImmutableIntList fields, - List<List<List<String>>> conditions, Schema schema, Runnable onClose) { + @Nullable Projector projector, @Nullable Filter filter, + Runnable onClose) { this.arrowFileReader = arrowFileReader; - this.conditions = conditions; - this.schema = schema; + this.projector = projector; + this.filter = filter; this.fields = fields; this.onClose = onClose; } @Override public Enumerator<Object> enumerator() { try { - if (!conditions.isEmpty()) { - return new ArrowFilterEnumerator(arrowFileReader, fields, - conditions, schema, onClose); + if (projector != null) { + return new ArrowProjectEnumerator(arrowFileReader, fields, projector, + onClose); + } else if (filter != null) { + return new ArrowFilterEnumerator(arrowFileReader, fields, filter, + onClose); } // No projector and no filter means the query is an identity projection // that should read selected value-vectors directly. diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java index 2f154cc6ef..5eddec2249 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java @@ -19,215 +19,91 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.gandiva.evaluator.Filter; +import org.apache.arrow.gandiva.evaluator.SelectionVector; +import org.apache.arrow.gandiva.evaluator.SelectionVectorInt16; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; + +import org.checkerframework.checker.nullness.qual.Nullable; import java.io.IOException; -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; import static java.util.Objects.requireNonNull; /** - * Enumerator that evaluates Arrow filter tokens in Java. + * Enumerator that reads from a filtered collection of Arrow value-vectors. */ class ArrowFilterEnumerator extends AbstractArrowEnumerator { - private final List<List<ConditionToken>> conditions; - private final Schema schema; + private final BufferAllocator allocator; + private final Filter filter; + private @Nullable ArrowBuf buf; + private @Nullable SelectionVector selectionVector; + private int selectionVectorIndex; + private final Runnable onClose; - private final List<ValueVector> filterVectors; - private final Map<String, Pattern> likePatterns; - ArrowFilterEnumerator(ArrowFileReader arrowFileReader, - ImmutableIntList fields, List<List<List<String>>> conditions, - Schema schema, Runnable onClose) { + ArrowFilterEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, + Filter filter, Runnable onClose) { super(arrowFileReader, fields); - this.conditions = toConditionTokens(conditions); - this.schema = schema; + this.allocator = new RootAllocator(Long.MAX_VALUE); + this.filter = filter; this.onClose = onClose; - this.filterVectors = new ArrayList<>(schema.getFields().size()); - this.likePatterns = new HashMap<>(); } - @Override protected void loadNextArrowBatch() { - super.loadNextArrowBatch(); - final VectorSchemaRoot root; + @Override void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { try { - root = arrowFileReader.getVectorSchemaRoot(); - } catch (IOException e) { + this.buf = this.allocator.buffer((long) rowCount * 2); + this.selectionVector = new SelectionVectorInt16(buf); + filter.evaluate(arrowRecordBatch, selectionVector); + } catch (GandivaException e) { throw Util.toUnchecked(e); } - filterVectors.clear(); - for (int i = 0; i < schema.getFields().size(); i++) { - filterVectors.add(root.getVector(i)); - } } @Override public boolean moveNext() { - while (true) { - if (currRowIndex >= rowCount - 1) { - if (!loadNextNonEmptyArrowBatch()) { - return false; + if (selectionVector == null + || selectionVectorIndex >= selectionVector.getRecordCount()) { + boolean hasNextBatch; + while (true) { + try { + hasNextBatch = arrowFileReader.loadNextBatch(); + } catch (IOException e) { + throw Util.toUnchecked(e); } - } - currRowIndex++; - if (matches(currRowIndex)) { - return true; - } - } - } - - private boolean matches(int rowIndex) { - for (List<ConditionToken> orGroup : conditions) { - boolean any = false; - for (ConditionToken token : orGroup) { - if (matches(token, rowIndex)) { - any = true; - break; + if (hasNextBatch) { + selectionVectorIndex = 0; + this.valueVectors.clear(); + loadNextArrowBatch(); + requireNonNull(selectionVector, "selectionVector"); + if (selectionVectorIndex >= selectionVector.getRecordCount()) { + // the "filtered" batch is empty, but there may be more batches to fetch + continue; + } + currRowIndex = selectionVector.getIndex(selectionVectorIndex++); } + return hasNextBatch; } - if (!any) { - return false; - } - } - return true; - } - - private boolean matches(ConditionToken token, int rowIndex) { - final Object value = getValue(fieldVector(token.fieldName), rowIndex); - switch (token.operator) { - case IS_NULL: - return value == null; - case IS_NOT_NULL: - return value != null; - case IS_TRUE: - return Boolean.TRUE.equals(value); - case IS_FALSE: - return Boolean.FALSE.equals(value); - case IS_NOT_TRUE: - return !Boolean.TRUE.equals(value); - case IS_NOT_FALSE: - return !Boolean.FALSE.equals(value); - case EQUAL: - return value != null && compare(value, literal(token)) == 0; - case NOT_EQUAL: - return value != null && compare(value, literal(token)) != 0; - case LESS_THAN: - return value != null && compare(value, literal(token)) < 0; - case LESS_THAN_OR_EQUAL: - return value != null && compare(value, literal(token)) <= 0; - case GREATER_THAN: - return value != null && compare(value, literal(token)) > 0; - case GREATER_THAN_OR_EQUAL: - return value != null && compare(value, literal(token)) >= 0; - case LIKE: - return value != null - && like(value.toString(), requireNonNull(token.value, "value")); - default: - throw new AssertionError("Unhandled Arrow filter operator: " + token.operator); - } - } - - private ValueVector fieldVector(String fieldName) { - final Field field = schema.findField(fieldName); - final int index = schema.getFields().indexOf(field); - if (index < 0) { - throw new IllegalArgumentException("Unknown Arrow field: " + fieldName); - } - return filterVectors.get(index); - } - - private static Object literal(ConditionToken token) { - final String type = requireNonNull(token.valueType, "valueType"); - final String value = requireNonNull(token.value, "value"); - if (type.startsWith("decimal")) { - return new BigDecimal(value); - } else if (type.equals("integer")) { - return Integer.valueOf(value); - } else if (type.equals("long")) { - return Long.valueOf(value); - } else if (type.equals("float")) { - return Float.valueOf(value); - } else if (type.equals("double")) { - return Double.valueOf(value); - } else if (type.equals("string")) { - return unquote(value); - } - throw new UnsupportedOperationException("Unsupported literal type: " + type); - } - - private static int compare(Object left, Object right) { - if (left instanceof BigDecimal || right instanceof BigDecimal) { - return toBigDecimal(left).compareTo(toBigDecimal(right)); - } - if (left instanceof Number && right instanceof Number) { - return Double.compare(((Number) left).doubleValue(), - ((Number) right).doubleValue()); - } - return left.toString().compareTo(right.toString()); - } - - private static BigDecimal toBigDecimal(Object value) { - if (value instanceof BigDecimal) { - return (BigDecimal) value; - } - return new BigDecimal(value.toString()); - } - - private boolean like(String value, String pattern) { - final String unquotedPattern = unquote(pattern); - final Pattern compiledPattern = - likePatterns.computeIfAbsent(unquotedPattern, p -> { - return Pattern.compile(toRegex(p), Pattern.DOTALL); - }); - return compiledPattern.matcher(value).matches(); - } - - private static String toRegex(String pattern) { - final StringBuilder builder = new StringBuilder(); - for (int i = 0; i < pattern.length(); i++) { - final char c = pattern.charAt(i); - if (c == '%') { - builder.append(".*"); - } else if (c == '_') { - builder.append('.'); - } else { - builder.append(Pattern.quote(String.valueOf(c))); - } + } else { + currRowIndex = selectionVector.getIndex(selectionVectorIndex++); + return true; } - return builder.toString(); } - private static String unquote(String value) { - if (value.length() >= 2 && value.charAt(0) == '\'' - && value.charAt(value.length() - 1) == '\'') { - return value.substring(1, value.length() - 1).replace("''", "'"); - } - return value; - } - - private static List<List<ConditionToken>> toConditionTokens( - List<List<List<String>>> conditions) { - final List<List<ConditionToken>> result = - new ArrayList<>(conditions.size()); - for (List<List<String>> orGroup : conditions) { - final List<ConditionToken> tokens = new ArrayList<>(orGroup.size()); - for (List<String> token : orGroup) { - tokens.add(ConditionToken.fromTokenList(token)); + @Override public void close() { + try { + if (buf != null) { + buf.close(); } - result.add(tokens); + filter.close(); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } finally { + onClose.run(); } - return result; - } - - @Override public void close() { - onClose.run(); } } diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java new file mode 100644 index 0000000000..0895f36cf1 --- /dev/null +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.arrow; + +import org.apache.calcite.util.ImmutableIntList; +import org.apache.calcite.util.Util; + +import org.apache.arrow.gandiva.evaluator.Projector; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; + +import java.io.IOException; + +/** + * Enumerator that reads from a projected collection of Arrow value-vectors. + */ +class ArrowProjectEnumerator extends AbstractArrowEnumerator { + private final Projector projector; + private final Runnable onClose; + + ArrowProjectEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList fields, + Projector projector, Runnable onClose) { + super(arrowFileReader, fields); + this.projector = projector; + this.onClose = onClose; + } + + @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch) { + try { + projector.evaluate(arrowRecordBatch, valueVectors); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } + } + + @Override public boolean moveNext() { + if (currRowIndex >= rowCount - 1) { + final boolean hasNextBatch; + try { + hasNextBatch = arrowFileReader.loadNextBatch(); + } catch (IOException e) { + throw Util.toUnchecked(e); + } + if (hasNextBatch) { + currRowIndex = 0; + this.valueVectors.clear(); + loadNextArrowBatch(); + } + return hasNextBatch; + } else { + currRowIndex++; + return true; + } + } + + @Override public void close() { + try { + projector.close(); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } finally { + onClose.run(); + } + } +} diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java index 3da1527f10..6e268d6469 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java @@ -100,7 +100,7 @@ RelNode convert(Filter filter) { final RelTraitSet traitSet = filter.getTraitSet().replace(ArrowRel.CONVENTION); // Expand SEARCH (e.g. IN, BETWEEN) before pushing to Arrow, - // since the Arrow adapter does not support SEARCH natively. + // since Gandiva does not support SEARCH natively. final RexNode condition = RexUtil.expandSearch(filter.getCluster().getRexBuilder(), null, filter.getCondition()); return new ArrowFilter(filter.getCluster(), traitSet, diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java index 2585f5d156..74438efe2a 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java @@ -38,9 +38,17 @@ import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; +import org.apache.arrow.gandiva.evaluator.Filter; +import org.apache.arrow.gandiva.evaluator.Projector; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.expression.Condition; +import org.apache.arrow.gandiva.expression.ExpressionTree; +import org.apache.arrow.gandiva.expression.TreeBuilder; +import org.apache.arrow.gandiva.expression.TreeNode; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.SeekableReadChannel; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -50,15 +58,20 @@ import java.io.FileInputStream; import java.io.IOException; import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.List; +import static java.lang.Double.parseDouble; +import static java.lang.Float.parseFloat; +import static java.lang.Integer.parseInt; +import static java.lang.Long.parseLong; import static java.util.Objects.requireNonNull; /** * Table backed by an Apache Arrow file. * - * <p>Reads data from an Arrow IPC file on disk. Projections and filters read - * directly from Arrow value-vectors. + * <p>Reads data from an Arrow IPC file on disk and supports projection + * and filter push-down via the Gandiva expression compiler. * * <p>Implements {@link TranslatableTable} so that it can be converted into * an {@link ArrowTableScan} for query planning, and {@link QueryableTable} @@ -103,6 +116,43 @@ public class ArrowTable extends AbstractTable public Enumerable<Object> query(DataContext root, ImmutableIntList fields, List<List<List<String>>> conditions) { requireNonNull(fields, "fields"); + final Projector projector; + final Filter filter; + + if (conditions.isEmpty()) { + filter = null; + projector = makeProjector(fields); + } else { + projector = null; + + final List<TreeNode> conjuncts = new ArrayList<>(conditions.size()); + for (List<List<String>> orGroup : conditions) { + final List<TreeNode> disjuncts = new ArrayList<>(orGroup.size()); + for (List<String> conditionParts : orGroup) { + disjuncts.add( + convertConditionToGandiva( + ConditionToken.fromTokenList(conditionParts))); + } + if (disjuncts.size() == 1) { + conjuncts.add(disjuncts.get(0)); + } else { + conjuncts.add(TreeBuilder.makeOr(disjuncts)); + } + } + final Condition filterCondition; + if (conjuncts.size() == 1) { + filterCondition = TreeBuilder.makeCondition(conjuncts.get(0)); + } else { + filterCondition = + TreeBuilder.makeCondition(TreeBuilder.makeAnd(conjuncts)); + } + + try { + filter = Filter.make(schema, filterCondition); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } + } FileInputStream fis = null; try { @@ -113,7 +163,7 @@ public Enumerable<Object> query(DataContext root, ImmutableIntList fields, final FileInputStream fisRef = fis; final Runnable onClose = () -> closeSilently(fisRef); fis = null; // ownership transferred to onClose - return new ArrowEnumerable(reader, fields, conditions, schema, onClose); + return new ArrowEnumerable(reader, fields, projector, filter, onClose); } catch (IOException e) { throw Util.toUnchecked(e); } finally { @@ -152,6 +202,70 @@ private static RelDataType deduceRowType(Schema schema, return builder.build(); } + private @Nullable Projector makeProjector(ImmutableIntList fields) { + if (requiresDirectVectorProjection(fields)) { + // Returning null selects ArrowEnumerable's direct vector-read path. + // Use that path because Gandiva does not support identity projection + // expressions over Arrow List and binary vectors. + return null; + } + + final List<ExpressionTree> expressionTrees = new ArrayList<>(); + for (int fieldOrdinal : fields) { + Field field = schema.getFields().get(fieldOrdinal); + TreeNode node = TreeBuilder.makeField(field); + expressionTrees.add(TreeBuilder.makeExpression(node, field)); + } + try { + return Projector.make(schema, expressionTrees); + } catch (GandivaException e) { + throw Util.toUnchecked(e); + } + } + + /** Returns whether selected fields should be projected by reading Arrow + * value-vectors directly rather than by creating a Gandiva projector. + * + * <p>CALCITE-7541 extends this direct projection path for Arrow binary vector + * families because Gandiva cannot project them through the existing identity + * projection path. Queries with filters still use Gandiva filters; this direct + * path only applies to no-filter projections. + */ + private boolean requiresDirectVectorProjection(ImmutableIntList fields) { + for (int fieldOrdinal : fields) { + switch (schema.getFields().get(fieldOrdinal).getType().getTypeID()) { + case List: + case Binary: + case LargeBinary: + case FixedSizeBinary: + return true; + default: + break; + } + } + return false; + } + + /** Converts a single {@link ConditionToken} into a Gandiva {@link TreeNode}. */ + private TreeNode convertConditionToGandiva(ConditionToken token) { + final List<TreeNode> treeNodes = new ArrayList<>(2); + treeNodes.add( + TreeBuilder.makeField(schema.getFields() + .get( + schema.getFields().indexOf( + schema.findField(token.fieldName))))); + + if (token.isBinary()) { + treeNodes.add( + makeLiteralNode( + requireNonNull(token.value, "value"), + requireNonNull(token.valueType, "valueType"))); + } + + return TreeBuilder.makeFunction( + token.operator, treeNodes, new ArrowType.Bool()); + } + /** Closes an {@link AutoCloseable} without throwing. */ private static void closeSilently(AutoCloseable closeable) { try { @@ -161,6 +275,29 @@ private static void closeSilently(AutoCloseable closeable) { } } + private static TreeNode makeLiteralNode(String literal, String type) { + if (type.startsWith("decimal")) { + String[] typeParts = + type.substring(type.indexOf('(') + 1, type.indexOf(')')).split(","); + int precision = parseInt(typeParts[0]); + int scale = parseInt(typeParts[1]); + return TreeBuilder.makeDecimalLiteral(literal, precision, scale); + } else if (type.equals("integer")) { + return TreeBuilder.makeLiteral(parseInt(literal)); + } else if (type.equals("long")) { + return TreeBuilder.makeLiteral(parseLong(literal)); + } else if (type.equals("float")) { + return TreeBuilder.makeLiteral(parseFloat(literal)); + } else if (type.equals("double")) { + return TreeBuilder.makeLiteral(parseDouble(literal)); + } else if (type.equals("string")) { + return TreeBuilder.makeStringLiteral(literal.substring(1, literal.length() - 1)); + } else { + throw new IllegalArgumentException("Invalid literal " + literal + + ", type " + type); + } + } + /** * Implementation of {@link Queryable} based on a {@link ArrowTable}. * diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java index 2b42295981..0ec6804052 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java @@ -35,26 +35,13 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.EQUAL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN_OR_EQUAL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_FALSE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_FALSE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_NULL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_TRUE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NULL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_TRUE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN_OR_EQUAL; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LIKE; -import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.NOT_EQUAL; import static org.apache.calcite.util.DateTimeStringUtils.ISO_DATETIME_FRACTIONAL_SECOND_FORMAT; import static org.apache.calcite.util.DateTimeStringUtils.getDateFormatter; import static java.util.Objects.requireNonNull; /** - * Translates a {@link RexNode} expression to Arrow predicate tokens. + * Translates a {@link RexNode} expression to Gandiva predicate tokens. */ class ArrowTranslator { final RexBuilder rexBuilder; @@ -78,7 +65,7 @@ public static ArrowTranslator create(RexBuilder rexBuilder, * * <p>If exceeded, {@link RexUtil#toCnf(RexBuilder, int, RexNode)} returns * the original expression unchanged, which may cause the subsequent - * translation to Arrow predicates to fail with an + * translation to Gandiva predicates to fail with an * {@link UnsupportedOperationException}. When invoked by the Arrow adapter * module, the exception is caught and the plan falls back to * an Enumerable convention. */ @@ -133,32 +120,32 @@ private static Object literalValue(RexLiteral literal) { private ConditionToken translateMatch2(RexNode node) { switch (node.getKind()) { case EQUALS: - return translateBinary(EQUAL, EQUAL, (RexCall) node); + return translateBinary("equal", "=", (RexCall) node); case NOT_EQUALS: - return translateBinary(NOT_EQUAL, NOT_EQUAL, (RexCall) node); + return translateBinary("not_equal", "<>", (RexCall) node); case LESS_THAN: - return translateBinary(LESS_THAN, GREATER_THAN, (RexCall) node); + return translateBinary("less_than", ">", (RexCall) node); case LESS_THAN_OR_EQUAL: - return translateBinary(LESS_THAN_OR_EQUAL, GREATER_THAN_OR_EQUAL, (RexCall) node); + return translateBinary("less_than_or_equal_to", ">=", (RexCall) node); case GREATER_THAN: - return translateBinary(GREATER_THAN, LESS_THAN, (RexCall) node); + return translateBinary("greater_than", "<", (RexCall) node); case GREATER_THAN_OR_EQUAL: - return translateBinary(GREATER_THAN_OR_EQUAL, LESS_THAN_OR_EQUAL, (RexCall) node); + return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node); case IS_NULL: - return translateUnary(IS_NULL, (RexCall) node); + return translateUnary("isnull", (RexCall) node); case IS_NOT_NULL: - return translateUnary(IS_NOT_NULL, (RexCall) node); + return translateUnary("isnotnull", (RexCall) node); case IS_NOT_TRUE: - return translateUnary(IS_NOT_TRUE, (RexCall) node); + return translateUnary("isnottrue", (RexCall) node); case IS_NOT_FALSE: - return translateUnary(IS_NOT_FALSE, (RexCall) node); + return translateUnary("isnotfalse", (RexCall) node); case INPUT_REF: final RexInputRef inputRef = (RexInputRef) node; - return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), IS_TRUE); + return ConditionToken.unary(fieldNames.get(inputRef.getIndex()), "istrue"); case NOT: - return translateUnary(IS_FALSE, (RexCall) node); + return translateUnary("isfalse", (RexCall) node); case LIKE: - return translateBinaryNoReverse(LIKE, (RexCall) node); + return translateBinary("like", null, (RexCall) node); default: throw new UnsupportedOperationException("Unsupported operator " + node); } @@ -168,8 +155,7 @@ private ConditionToken translateMatch2(RexNode node) { * Translates a call to a binary operator, reversing arguments if * necessary. */ - private ConditionToken translateBinary(ConditionToken.Operator op, - ConditionToken.Operator rop, RexCall call) { + private ConditionToken translateBinary(String op, String rop, RexCall call) { final RexNode left = call.operands.get(0); final RexNode right = call.operands.get(1); @Nullable ConditionToken expression = translateBinary2(op, left, right); @@ -183,21 +169,9 @@ private ConditionToken translateBinary(ConditionToken.Operator op, throw new UnsupportedOperationException("Unsupported binary operator " + call); } - /** Translates a call to a binary operator without reversing arguments. */ - private ConditionToken translateBinaryNoReverse(ConditionToken.Operator op, - RexCall call) { - final RexNode left = call.operands.get(0); - final RexNode right = call.operands.get(1); - @Nullable ConditionToken expression = translateBinary2(op, left, right); - if (expression != null) { - return expression; - } - throw new UnsupportedOperationException("Unsupported binary operator " + call); - } - /** Translates a call to a binary operator. Returns null on failure. */ - private @Nullable ConditionToken translateBinary2( - ConditionToken.Operator op, RexNode left, RexNode right) { + private @Nullable ConditionToken translateBinary2(String op, RexNode left, + RexNode right) { if (right.getKind() != SqlKind.LITERAL) { return null; } @@ -217,7 +191,7 @@ private ConditionToken translateBinaryNoReverse(ConditionToken.Operator op, /** Combines a field name, operator, and literal to produce a binary * condition token. */ - private ConditionToken translateOp2(ConditionToken.Operator op, String name, + private ConditionToken translateOp2(String op, String name, RexLiteral right) { Object value = literalValue(right); String valueString = value.toString(); @@ -235,7 +209,7 @@ private ConditionToken translateOp2(ConditionToken.Operator op, String name, } /** Translates a call to a unary operator. */ - private ConditionToken translateUnary(ConditionToken.Operator op, RexCall call) { + private ConditionToken translateUnary(String op, RexCall call) { final RexNode opNode = call.operands.get(0); @Nullable ConditionToken expression = translateUnary2(op, opNode); @@ -247,8 +221,7 @@ private ConditionToken translateUnary(ConditionToken.Operator op, RexCall call) } /** Translates a call to a unary operator. Returns null on failure. */ - private @Nullable ConditionToken translateUnary2(ConditionToken.Operator op, - RexNode opNode) { + private @Nullable ConditionToken translateUnary2(String op, RexNode opNode) { if (opNode.getKind() == SqlKind.INPUT_REF) { final RexInputRef inputRef = (RexInputRef) opNode; final String name = fieldNames.get(inputRef.getIndex()); diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java index c5b690840a..44d3facea7 100644 --- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java +++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java @@ -25,7 +25,7 @@ import static java.util.Objects.requireNonNull; /** - * A structured representation of a single Arrow predicate condition. + * A structured representation of a single Gandiva predicate condition. * * <p>A condition is either unary (e.g. {@code IS NULL}) or binary * (e.g. {@code =}, {@code <}). Unary conditions have a field name @@ -36,11 +36,11 @@ */ class ConditionToken { final String fieldName; - final Operator operator; + final String operator; final @Nullable String value; final @Nullable String valueType; - private ConditionToken(String fieldName, Operator operator, + private ConditionToken(String fieldName, String operator, @Nullable String value, @Nullable String valueType) { this.fieldName = requireNonNull(fieldName, "fieldName"); this.operator = requireNonNull(operator, "operator"); @@ -50,7 +50,7 @@ private ConditionToken(String fieldName, Operator operator, /** Creates a binary condition token * (e.g. {@code intField equal 12 integer}). */ - static ConditionToken binary(String fieldName, Operator operator, + static ConditionToken binary(String fieldName, String operator, String value, String valueType) { return new ConditionToken(fieldName, operator, requireNonNull(value, "value"), @@ -59,7 +59,7 @@ static ConditionToken binary(String fieldName, Operator operator, /** Creates a unary condition token * (e.g. {@code intField isnull}). */ - static ConditionToken unary(String fieldName, Operator operator) { + static ConditionToken unary(String fieldName, String operator) { return new ConditionToken(fieldName, operator, null, null); } @@ -76,55 +76,22 @@ boolean isBinary() { * binary conditions. */ List<String> toTokenList() { if (isBinary()) { - return ImmutableList.of(fieldName, operator.token, + return ImmutableList.of(fieldName, operator, requireNonNull(value, "value"), requireNonNull(valueType, "valueType")); } - return ImmutableList.of(fieldName, operator.token); + return ImmutableList.of(fieldName, operator); } /** Creates a {@code ConditionToken} from a serialized string list. */ static ConditionToken fromTokenList(List<String> tokens) { final int size = tokens.size(); if (size == 4) { - return binary(tokens.get(0), Operator.of(tokens.get(1)), + return binary(tokens.get(0), tokens.get(1), tokens.get(2), tokens.get(3)); } else if (size == 2) { - return unary(tokens.get(0), Operator.of(tokens.get(1))); + return unary(tokens.get(0), tokens.get(1)); } throw new IllegalArgumentException("Invalid condition tokens: " + tokens); } - - /** Operators supported by the Arrow adapter filter representation. */ - enum Operator { - IS_NULL("isnull"), - IS_NOT_NULL("isnotnull"), - IS_TRUE("istrue"), - IS_FALSE("isfalse"), - IS_NOT_TRUE("isnottrue"), - IS_NOT_FALSE("isnotfalse"), - EQUAL("equal"), - NOT_EQUAL("not_equal"), - LESS_THAN("less_than"), - LESS_THAN_OR_EQUAL("less_than_or_equal_to"), - GREATER_THAN("greater_than"), - GREATER_THAN_OR_EQUAL("greater_than_or_equal_to"), - LIKE("like"); - - final String token; - - Operator(String token) { - this.token = token; - } - - static Operator of(String token) { - for (Operator operator : values()) { - if (operator.token.equals(token)) { - return operator; - } - } - throw new UnsupportedOperationException( - "Unsupported Arrow filter operator: " + token); - } - } } diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java index 8c9fda7f08..275bdd0be7 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java @@ -82,11 +82,6 @@ static void initializeArrowState(@TempDir Path sharedTempDir) arrowDataGenerator.writeArrowData(dataLocationFile); arrowDataGenerator.writeScottEmpData(arrowFilesDirectory); - File emptyBatchDataLocationFile = - arrowFilesDirectory.resolve("arrowemptybatch.arrow").toFile(); - ArrowDataTest emptyBatchDataGenerator = new ArrowDataTest(); - emptyBatchDataGenerator.writeArrowDataWithEmptyBatch(emptyBatchDataLocationFile); - File datatypeLocationFile = arrowFilesDirectory.resolve("arrowdatatype.arrow").toFile(); ArrowDataTest arrowtypeDataGenerator = new ArrowDataTest(); arrowtypeDataGenerator.writeArrowDataType(datatypeLocationFile); @@ -265,51 +260,6 @@ static void initializeArrowState(@TempDir Path sharedTempDir) .explainContains(plan); } - /** Test case for - * <a href="https://issues.apache.org/jira/browse/CALCITE-7580";>[CALCITE-7580] - * Remove Gandiva dependency from Arrow adapter</a>. */ - @Test void testArrowProjectSkipsEmptyBatch() { - String sql = "select \"intField\", \"stringField\" from arrowemptybatch\n"; - String result = "intField=0; stringField=0\n" - + "intField=1; stringField=1\n" - + "intField=2; stringField=2\n"; - - CalciteAssert.that() - .with(arrow) - .query(sql) - .returns(result); - } - - /** Test case for - * <a href="https://issues.apache.org/jira/browse/CALCITE-7580";>[CALCITE-7580] - * Remove Gandiva dependency from Arrow adapter</a>. */ - @Test void testArrowFilterSkipsEmptyBatch() { - String sql = "select \"intField\", \"stringField\"\n" - + "from arrowemptybatch\n" - + "where \"intField\" > 0"; - String result = "intField=1; stringField=1\n" - + "intField=2; stringField=2\n"; - - CalciteAssert.that() - .with(arrow) - .query(sql) - .returns(result); - } - - /** Test case for - * <a href="https://issues.apache.org/jira/browse/CALCITE-7580";>[CALCITE-7580] - * Remove Gandiva dependency from Arrow adapter</a>. */ - @Test void testArrowFilterSkipsEmptyBatchWithNoMatches() { - String sql = "select \"intField\", \"stringField\"\n" - + "from arrowemptybatch\n" - + "where \"intField\" < 0"; - - CalciteAssert.that() - .with(arrow) - .query(sql) - .returns(""); - } - @Test void testArrowProjectFieldsWithIntegerFilter() { String sql = "select \"intField\", \"stringField\"\n" + "from arrowdata\n" diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java index 4b9af441aa..a53cc1231a 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java @@ -301,35 +301,6 @@ public void writeArrowBinaryData(File file) throws IOException { fileOutputStream.close(); } - public void writeArrowDataWithEmptyBatch(File file) throws IOException { - Schema arrowSchema = makeArrowSchema(); - try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE); - VectorSchemaRoot vectorSchemaRoot = - VectorSchemaRoot.create(arrowSchema, allocator); - FileOutputStream fileOutputStream = new FileOutputStream(file); - ArrowFileWriter arrowFileWriter = - new ArrowFileWriter(vectorSchemaRoot, null, - fileOutputStream.getChannel())) { - arrowFileWriter.start(); - - vectorSchemaRoot.setRowCount(0); - for (Field field : vectorSchemaRoot.getSchema().getFields()) { - vectorSchemaRoot.getVector(field.getName()).setValueCount(0); - } - arrowFileWriter.writeBatch(); - - int rowCount = 3; - vectorSchemaRoot.setRowCount(rowCount); - intField(vectorSchemaRoot.getVector("intField"), rowCount); - varCharField(vectorSchemaRoot.getVector("stringField"), rowCount); - floatField(vectorSchemaRoot.getVector("floatField"), rowCount); - longField(vectorSchemaRoot.getVector("longField"), rowCount); - arrowFileWriter.writeBatch(); - - arrowFileWriter.end(); - } - } - public void writeArrowDataType(File file) throws IOException { FileOutputStream fileOutputStream = new FileOutputStream(file); Schema arrowSchema = makeArrowDateTypeSchema(); diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java index 4600dab100..ab1f5c2a88 100644 --- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java +++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java @@ -18,12 +18,22 @@ import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.arrow.gandiva.evaluator.Projector; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.expression.ExpressionTree; +import org.apache.arrow.vector.types.pojo.Schema; + import org.junit.jupiter.api.extension.ConditionEvaluationResult; import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtensionContext; +import java.util.ArrayList; +import java.util.List; + /** * JUnit5 extension to handle Arrow tests. + * + * <p>Tests will be skipped if the Gandiva library cannot be loaded on the given platform. */ class ArrowExtension implements ExecutionCondition { @@ -31,7 +41,8 @@ class ArrowExtension implements ExecutionCondition { * Whether to run this test. * * <p>Enabled by default, unless explicitly disabled from command line - * ({@code -Dcalcite.test.arrow=false}). + * ({@code -Dcalcite.test.arrow=false}) or if Gandiva library, used to implement arrow + * filtering/projection, cannot be loaded. * * @return {@code true} if the test is enabled and can run in the current environment, * {@code false} otherwise @@ -40,6 +51,16 @@ class ArrowExtension implements ExecutionCondition { final ExtensionContext context) { boolean enabled = CalciteSystemProperty.TEST_ARROW.value(); + try { + Schema emptySchema = new Schema(new ArrayList<>(), null); + List<ExpressionTree> expressions = new ArrayList<>(); + Projector.make(emptySchema, expressions); + } catch (GandivaException e) { + // this exception comes from using an empty expression, + // but the JNI library was loaded properly + } catch (UnsatisfiedLinkError e) { + enabled = false; + } if (enabled) { return ConditionEvaluationResult.enabled("Arrow tests enabled"); diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 64cb532afd..f00ed7d8e5 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -101,6 +101,7 @@ fun DependencyConstraintHandlerScope.runtimev( apiv("org.apache.arrow:arrow-memory-netty", "arrow") apiv("org.apache.arrow:arrow-vector", "arrow") apiv("org.apache.arrow:arrow-jdbc", "arrow") + apiv("org.apache.arrow.gandiva:arrow-gandiva", "arrow-gandiva") apiv("org.apache.calcite.avatica:avatica-core", "calcite.avatica") apiv("org.apache.calcite.avatica:avatica-server", "calcite.avatica") apiv("org.apache.cassandra:cassandra-all") diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index 4d7702295b..0ffb60454e 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -529,11 +529,14 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) { } if (e.operands.size() == 3 && e.operands.get(2) instanceof RexLiteral) { final RexLiteral escapeLiteral = (RexLiteral) e.operands.get(2); - Character escape = requireNonNull(escapeLiteral.getValueAs(Character.class)); - e = (RexCall) rexBuilder - .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), - rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%'), - e.operands.get(1).getType(), true, true), escapeLiteral); + final String escapeStr = requireNonNull(escapeLiteral.getValueAs(String.class)); + if (escapeStr.length() == 1) { + char escape = escapeStr.charAt(0); + e = (RexCall) rexBuilder + .makeCall(e.getParserPosition(), e.getOperator(), e.operands.get(0), + rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape, '%'), + e.operands.get(1).getType(), true, true), escapeLiteral); + } } } return simplifyGenericNode(e); @@ -543,7 +546,7 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) { // string with even escapes 'AA\\\\%%__%%AA' simplify to 'AA\\__%AA' // string with odd escapes 'AA\\\\\\%%__%%AA' simplify to 'AA\\\\\\%__%AA' private String simplifyMixedWildcards(String str, char escape) { - Pattern pattern = Pattern.compile("[_%]+"); + Pattern pattern = getWildCardPattern(escape); Matcher matcher = pattern.matcher(str); StringBuilder builder = new StringBuilder(); int from = 0; @@ -567,6 +570,17 @@ && consecutiveSameCharCountBefore(str, start - 1, escape) % 2 == 1) { return builder.toString(); } + private static Pattern getWildCardPattern(char escape) { + switch (escape) { + case '%': + return Pattern.compile("_+"); + case '_': + return Pattern.compile("%+"); + default: + return Pattern.compile("[_%]+"); + } + } + // Tool method: count the number of consecutive identical characters before index private int consecutiveSameCharCountBefore(String str, int index, char escape) { int count = 0; diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java index 115b66fa21..d0ccab7dfd 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java @@ -561,7 +561,7 @@ private static void assertBasic(SqlTypeName typeName) { } } - if (type.getSqlTypeName() == resultType.getSqlTypeName() + if (type.getSqlTypeName().getFamily() == resultType.getSqlTypeName().getFamily() && type.getSqlTypeName().allowsPrec() && type.getPrecision() != resultType.getPrecision()) { final int precision = diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 4d14e16937..882d423870 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -1870,7 +1870,7 @@ public RelNode convertToSingleValueSubq( if (leftKeys.size() == 1) { SqlCall sqlCall = comparisonOp.createCall(rightVals.getParserPosition(), leftKeys.get(0), rightVals); - rexComparison = bb.convertExpression(sqlCall); + rexComparison = ensureComparisonTypes(bb.convertExpression(sqlCall)); } else { assert rightVals instanceof SqlCall; final SqlBasicCall call = (SqlBasicCall) rightVals; @@ -1880,9 +1880,10 @@ public RelNode convertToSingleValueSubq( RexUtil.composeConjunction(rexBuilder, transform( Pair.zip(leftKeys, call.getOperandList()), - pair -> bb.convertExpression( - comparisonOp.createCall(rightVals.getParserPosition(), - pair.left, pair.right)))); + pair -> ensureComparisonTypes( + bb.convertExpression( + comparisonOp.createCall(rightVals.getParserPosition(), + pair.left, pair.right))))); } comparisons.add(rexComparison); } @@ -1901,6 +1902,31 @@ public RelNode convertToSingleValueSubq( } } + /** + * Ensures that a comparison expression has matching operand types. If the + * operands have different type names, casts the right operand to match the + * left operand's type. This handles the case where type coercion is disabled + * and the IN-to-OR expansion produces comparisons with mismatched types + * (e.g., DATE = CHAR). + */ + private RexNode ensureComparisonTypes(RexNode node) { + if (validator != null && validator.config().typeCoercionEnabled()) { + return node; + } + if (node instanceof RexCall) { + final RexCall call = (RexCall) node; + if (call.operands.size() == 2) { + final RexNode left = call.operands.get(0); + final RexNode right = call.operands.get(1); + if (left.getType().getSqlTypeName() != right.getType().getSqlTypeName()) { + final RexNode castRight = rexBuilder.ensureType(left.getType(), right, true); + return rexBuilder.makeCall(call.getOperator(), left, castRight); + } + } + } + return node; + } + /** * Converts a {@link SqlNodeList} (for example an IN-list or VALUES list) * into a relational expression and produces a Rex-level sub-query that diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java index 2bdc7a1d56..ac7b5aebe6 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java @@ -4358,6 +4358,10 @@ private void checkSarg(String message, Sarg sarg, * Multiple consecutive '%' in the string matched by LIKE should simplify to a single '%'</a>, * <a href="https://issues.apache.org/jira/browse/CALCITE-7153";>[CALCITE-7153] * Mixed wildcards of _ and % need to be simplified in LIKE operator</a>. + * <a href="https://issues.apache.org/jira/browse/CALCITE-7578";>[CALCITE-7578] + * LIKE with empty ESCAPE might fail with StringIndexOutOfBoundsException</a>. + * <a href="https://issues.apache.org/jira/browse/CALCITE-7588";>[CALCITE-7588] + * LIKE with ESCAPE symbols containing wildcards fails</a>. * */ @Test void testSimplifyLike() { final RexNode ref = input(tVarchar(true, 10), 0); @@ -4421,6 +4425,14 @@ private void checkSarg(String message, Sarg sarg, "LIKE($0, '###%%#%#%A#%%#%A%###%%', '#')"); checkSimplifyUnchanged(like(ref, literal("A"), literal("#"))); checkSimplifyUnchanged(like(ref, literal("%A"), literal("#"))); + checkSimplifyUnchanged(like(ref, literal("TE%_ST"), literal("%"))); + checkSimplifyUnchanged(like(ref, literal("TE%%ST"), literal("%"))); + checkSimplifyUnchanged(like(ref, literal("a%_b%%c"), literal("%"))); + checkSimplifyUnchanged(like(ref, literal("%_%%A%_"), literal("%"))); + // escape char equal to the '_' wildcard. '%E__S%' ESCAPE '_' is a literal '_'. + checkSimplifyUnchanged(like(ref, literal("%E__S%"), literal("_"))); + checkSimplifyUnchanged(like(ref, literal("TE_%ST"), literal("_"))); + checkSimplifyUnchanged(like(ref, literal("a_%b__c"), literal("_"))); // As above, but ref is NOT NULL final RexNode refMandatory = vVarcharNotNull(0); @@ -4451,6 +4463,14 @@ private void checkSarg(String message, Sarg sarg, // NOT(SIMILAR TO) is not optimized checkSimplifyUnchanged( not(rexBuilder.makeCall(SqlStdOperatorTable.SIMILAR_TO, ref, literal("%")))); + + try { + // Empty ESCAPE + checkSimplifyUnchanged(like(ref, literal("a"), literal(""))); + } catch (RuntimeException e) { + assertThat(e.getMessage(), + containsString("Invalid escape character ''")); + } } @Test void testSimplifyNullCheckInFilter() { diff --git a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java index 8f5e5e4018..f0ff190d28 100644 --- a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java +++ b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java @@ -33,6 +33,7 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -172,6 +173,30 @@ class SqlTypeFactoryTest { assertThat(leastRestrictive.getPrecision(), is(3)); } + /** + * Test case for + * <a href="https://issues.apache.org/jira/browse/CALCITE-7567";> + * LeastRetrictiveSqlType for TIMESTAMP, TIMESTAMP_LTZ might ignore precision</a>. */ + @Test void testLeastRestrictiveForTimestampAndTimestampLtz() { + SqlTypeFixture f = new SqlTypeFixture(); + RelDataType ltz0 = + f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0); + RelDataType leastRestrictive = + f.typeFactory.leastRestrictive(Lists.newArrayList(ltz0, f.sqlTimestampPrec3)); + assertThat(leastRestrictive, is(notNullValue())); + assertThat(leastRestrictive.getPrecision(), is(3)); + } + + @Test void testLeastRestrictiveForTimestampLtzAndTimestamp() { + SqlTypeFixture f = new SqlTypeFixture(); + RelDataType ltz0 = + f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0); + RelDataType leastRestrictive = + f.typeFactory.leastRestrictive(Lists.newArrayList(f.sqlTimestampPrec3, ltz0)); + assertThat(leastRestrictive, is(notNullValue())); + assertThat(leastRestrictive.getPrecision(), is(3)); + } + @Test void testLeastRestrictiveForTimestampAndDate() { SqlTypeFixture f = new SqlTypeFixture(); RelDataType leastRestrictive = diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 694572683e..26000620af 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -2213,6 +2213,24 @@ void checkCorrelatedMapSubQuery(boolean expand) { sql(sql).withExpand(false).ok(); } + /** Test case for + * <a href="https://issues.apache.org/jira/browse/CALCITE-7562";>[CALCITE-7562] + * SqlToRel misses CAST in case IN expression without type coercion</a>. */ + @Test void testInDateColumnWithoutTypeCoercion() { + final String sql = + "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')"; + sql(sql).withTypeCoercion(false).ok(); + } + + /** Test case for + * <a href="https://issues.apache.org/jira/browse/CALCITE-7562";>[CALCITE-7562] + * SqlToRel misses CAST in case IN expression without type coercion</a>. */ + @Test void testInDateColumnWithTypeCoercion() { + final String sql = + "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')"; + sql(sql).ok(); + } + @Test void testInValueListLong() { // Go over the default threshold of 20 to force a sub-query. final String sql = "select empno from emp where deptno in" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 0bf02c9a73..71fe294d2f 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -3305,6 +3305,30 @@ LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) LogicalAggregate(group=[{0}], S=[SUM($1)], agg#1=[COUNT()]) LogicalProject(DEPTNO=[$7], SAL=[$5]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + </Resource> + </TestCase> + <TestCase name="testInDateColumnWithTypeCoercion"> + <Resource name="sql"> + <![CDATA[select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')]]> + </Resource> + <Resource name="plan"> + <![CDATA[ +LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], BIRTHDATE=[$9]) + LogicalFilter(condition=[OR(=($9, CAST('2000-06-30'):DATE NOT NULL), =($9, CAST('2000-09-27'):DATE NOT NULL))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) +]]> + </Resource> + </TestCase> + <TestCase name="testInDateColumnWithoutTypeCoercion"> + <Resource name="sql"> + <![CDATA[select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')]]> + </Resource> + <Resource name="plan"> + <![CDATA[ +LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], BIRTHDATE=[$9]) + LogicalFilter(condition=[OR(=($9, CAST('2000-06-30'):DATE NOT NULL), =($9, CAST('2000-09-27'):DATE NOT NULL))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP_B]]) ]]> </Resource> </TestCase> diff --git a/gradle.properties b/gradle.properties index fd82318264..eb0bd778ae 100644 --- a/gradle.properties +++ b/gradle.properties @@ -81,6 +81,7 @@ jandex.version=3.5.3 # elasticsearch does not like asm:6.2.1+ aggdesigner-algorithm.version=6.1 apiguardian-api.version=1.1.2 +arrow-gandiva.version=15.0.0 arrow.version=15.0.0 asm.version=9.9.1 byte-buddy.version=1.18.8 diff --git a/site/_docs/history.md b/site/_docs/history.md index c72d6f78d9..15e2561d39 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -49,11 +49,6 @@ ## <a href="https://github.com/apache/calcite/releases/tag/calcite-1.43.0";>1.43. #### Breaking Changes {: #breaking-1-43-0} -* [<a href="https://issues.apache.org/jira/browse/CALCITE-7580";>CALCITE-7580</a>] - Remove Gandiva dependency from Arrow adapter. Arrow adapter projection and - filter evaluation now run in Java, and the `arrow-gandiva` dependency is no - longer included in the Arrow module or BOM. - #### New features {: #new-features-1-43-0} diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java index a6685cf952..784cb1eba1 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java @@ -178,6 +178,11 @@ public SqlToRelFixture withConformance(SqlConformance conformance) { .withValidatorConfig(c -> c.withConformance(conformance))); } + public SqlToRelFixture withTypeCoercion(boolean enabled) { + return withFactory(f -> + f.withValidatorConfig(c -> c.withTypeCoercionEnabled(enabled))); + } + public SqlToRelFixture withDiffRepos(DiffRepository diffRepos) { return new SqlToRelFixture(sql, decorrelate, tester, factory, trim, expression, diffRepos);
