This is an automated email from the ASF dual-hosted git repository.
caicancai pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new bf2b81b27d [CALCITE-7580] Remove Gandiva dependency from Arrow adapter
bf2b81b27d is described below
commit bf2b81b27d76ef3571ba676dbc7bce4ccb4a0413
Author: Cancai Cai <[email protected]>
AuthorDate: Sat Jun 6 22:43:38 2026 +0800
[CALCITE-7580] Remove Gandiva dependency from Arrow adapter
---
arrow/build.gradle.kts | 1 -
.../adapter/arrow/AbstractArrowEnumerator.java | 30 ++-
.../adapter/arrow/ArrowDirectEnumerator.java | 31 +--
.../calcite/adapter/arrow/ArrowEnumerable.java | 25 +--
.../adapter/arrow/ArrowFilterEnumerator.java | 238 ++++++++++++++++-----
.../adapter/arrow/ArrowProjectEnumerator.java | 80 -------
.../apache/calcite/adapter/arrow/ArrowRules.java | 2 +-
.../apache/calcite/adapter/arrow/ArrowTable.java | 143 +------------
.../calcite/adapter/arrow/ArrowTranslator.java | 69 ++++--
.../calcite/adapter/arrow/ConditionToken.java | 51 ++++-
.../calcite/adapter/arrow/ArrowAdapterTest.java | 50 +++++
.../calcite/adapter/arrow/ArrowDataTest.java | 29 +++
.../calcite/adapter/arrow/ArrowExtension.java | 23 +-
bom/build.gradle.kts | 1 -
.../java/org/apache/calcite/rex/RexSimplify.java | 26 +--
.../calcite/sql/type/SqlTypeFactoryImpl.java | 2 +-
.../apache/calcite/sql2rel/SqlToRelConverter.java | 34 +--
.../org/apache/calcite/rex/RexProgramTest.java | 20 --
.../calcite/sql/type/SqlTypeFactoryTest.java | 25 ---
.../apache/calcite/test/SqlToRelConverterTest.java | 18 --
.../apache/calcite/test/SqlToRelConverterTest.xml | 24 ---
gradle.properties | 1 -
site/_docs/history.md | 5 +
.../org/apache/calcite/test/SqlToRelFixture.java | 5 -
24 files changed, 408 insertions(+), 525 deletions(-)
diff --git a/arrow/build.gradle.kts b/arrow/build.gradle.kts
index 598aa8a879..c75a8b5f67 100644
--- a/arrow/build.gradle.kts
+++ b/arrow/build.gradle.kts
@@ -23,7 +23,6 @@
implementation("com.google.guava:guava")
implementation("org.apache.arrow:arrow-memory-netty")
implementation("org.apache.arrow:arrow-vector")
- implementation("org.apache.arrow.gandiva:arrow-gandiva")
annotationProcessor("org.immutables:value")
compileOnly("org.immutables:value-annotations")
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
index e188757b0d..486e3f60bd 100644
---
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
+++
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
@@ -24,9 +24,7 @@
import org.apache.arrow.vector.TimeStampVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
import org.apache.arrow.vector.ipc.ArrowFileReader;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.types.TimeUnit;
import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -51,8 +49,6 @@ abstract class AbstractArrowEnumerator implements
Enumerator<Object> {
this.currRowIndex = -1;
}
- abstract void evaluateOperator(ArrowRecordBatch arrowRecordBatch);
-
protected void loadNextArrowBatch() {
try {
final VectorSchemaRoot vsr = arrowFileReader.getVectorSchemaRoot();
@@ -60,14 +56,32 @@ protected void loadNextArrowBatch() {
this.valueVectors.add(vsr.getVector(i));
}
this.rowCount = vsr.getRowCount();
- VectorUnloader vectorUnloader = new VectorUnloader(vsr);
- ArrowRecordBatch arrowRecordBatch = vectorUnloader.getRecordBatch();
- evaluateOperator(arrowRecordBatch);
} catch (IOException e) {
throw Util.toUnchecked(e);
}
}
+ /** Loads the next non-empty Arrow batch. */
+ protected boolean loadNextNonEmptyArrowBatch() {
+ while (true) {
+ final boolean hasNextBatch;
+ try {
+ hasNextBatch = arrowFileReader.loadNextBatch();
+ } catch (IOException e) {
+ throw Util.toUnchecked(e);
+ }
+ if (!hasNextBatch) {
+ return false;
+ }
+ currRowIndex = -1;
+ valueVectors.clear();
+ loadNextArrowBatch();
+ if (rowCount > 0) {
+ return true;
+ }
+ }
+ }
+
@Override public Object current() {
if (fields.size() == 1) {
return getValue(this.valueVectors.get(0), currRowIndex);
@@ -85,7 +99,7 @@ protected void loadNextArrowBatch() {
* <p>For {@link TimeStampVector}, converts the raw value to
* milliseconds since epoch, which is the representation used by
* Calcite's Enumerable runtime for TIMESTAMP types. */
- private static Object getValue(ValueVector vector, int index) {
+ protected static Object getValue(ValueVector vector, int index) {
if (vector instanceof TimeStampVector) {
if (vector.isNull(index)) {
return null;
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java
index 0cdec7baeb..787ffd88d9 100644
---
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java
+++
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java
@@ -17,19 +17,11 @@
package org.apache.calcite.adapter.arrow;
import org.apache.calcite.util.ImmutableIntList;
-import org.apache.calcite.util.Util;
import org.apache.arrow.vector.ipc.ArrowFileReader;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-
-import java.io.IOException;
/**
* Enumerator that reads projected Arrow value-vectors directly.
- *
- * <p>This path is used for identity projections that Gandiva cannot project
- * through the existing {@code Projector} path, such as Arrow binary vectors.
- * It is not a replacement for Gandiva expression evaluation.
*/
class ArrowDirectEnumerator extends AbstractArrowEnumerator {
private final Runnable onClose;
@@ -40,27 +32,14 @@ class ArrowDirectEnumerator extends AbstractArrowEnumerator
{
this.onClose = onClose;
}
- @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch)
{
- }
-
@Override public boolean moveNext() {
- if (currRowIndex >= rowCount - 1) {
- final boolean hasNextBatch;
- try {
- hasNextBatch = arrowFileReader.loadNextBatch();
- } catch (IOException e) {
- throw Util.toUnchecked(e);
- }
- if (hasNextBatch) {
- currRowIndex = 0;
- this.valueVectors.clear();
- loadNextArrowBatch();
+ while (currRowIndex >= rowCount - 1) {
+ if (!loadNextNonEmptyArrowBatch()) {
+ return false;
}
- return hasNextBatch;
- } else {
- currRowIndex++;
- return true;
}
+ currRowIndex++;
+ return true;
}
@Override public void close() {
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
index 84ed5997aa..b9c0c4171e 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
@@ -21,11 +21,10 @@
import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.Util;
-import org.apache.arrow.gandiva.evaluator.Filter;
-import org.apache.arrow.gandiva.evaluator.Projector;
import org.apache.arrow.vector.ipc.ArrowFileReader;
+import org.apache.arrow.vector.types.pojo.Schema;
-import org.checkerframework.checker.nullness.qual.Nullable;
+import java.util.List;
/**
* Enumerable that reads from Arrow value-vectors.
@@ -33,28 +32,24 @@
class ArrowEnumerable extends AbstractEnumerable<Object> {
private final ArrowFileReader arrowFileReader;
private final ImmutableIntList fields;
- private final @Nullable Projector projector;
- private final @Nullable Filter filter;
+ private final List<List<List<String>>> conditions;
+ private final Schema schema;
private final Runnable onClose;
ArrowEnumerable(ArrowFileReader arrowFileReader, ImmutableIntList fields,
- @Nullable Projector projector, @Nullable Filter filter,
- Runnable onClose) {
+ List<List<List<String>>> conditions, Schema schema, Runnable onClose) {
this.arrowFileReader = arrowFileReader;
- this.projector = projector;
- this.filter = filter;
+ this.conditions = conditions;
+ this.schema = schema;
this.fields = fields;
this.onClose = onClose;
}
@Override public Enumerator<Object> enumerator() {
try {
- if (projector != null) {
- return new ArrowProjectEnumerator(arrowFileReader, fields, projector,
- onClose);
- } else if (filter != null) {
- return new ArrowFilterEnumerator(arrowFileReader, fields, filter,
- onClose);
+ if (!conditions.isEmpty()) {
+ return new ArrowFilterEnumerator(arrowFileReader, fields,
+ conditions, schema, onClose);
}
// No projector and no filter means the query is an identity projection
// that should read selected value-vectors directly.
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java
index 5eddec2249..2f154cc6ef 100644
---
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java
+++
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java
@@ -19,91 +19,215 @@
import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.Util;
-import org.apache.arrow.gandiva.evaluator.Filter;
-import org.apache.arrow.gandiva.evaluator.SelectionVector;
-import org.apache.arrow.gandiva.evaluator.SelectionVectorInt16;
-import org.apache.arrow.gandiva.exceptions.GandivaException;
-import org.apache.arrow.memory.ArrowBuf;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowFileReader;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-
-import org.checkerframework.checker.nullness.qual.Nullable;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
import static java.util.Objects.requireNonNull;
/**
- * Enumerator that reads from a filtered collection of Arrow value-vectors.
+ * Enumerator that evaluates Arrow filter tokens in Java.
*/
class ArrowFilterEnumerator extends AbstractArrowEnumerator {
- private final BufferAllocator allocator;
- private final Filter filter;
- private @Nullable ArrowBuf buf;
- private @Nullable SelectionVector selectionVector;
- private int selectionVectorIndex;
-
+ private final List<List<ConditionToken>> conditions;
+ private final Schema schema;
private final Runnable onClose;
+ private final List<ValueVector> filterVectors;
+ private final Map<String, Pattern> likePatterns;
- ArrowFilterEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList
fields,
- Filter filter, Runnable onClose) {
+ ArrowFilterEnumerator(ArrowFileReader arrowFileReader,
+ ImmutableIntList fields, List<List<List<String>>> conditions,
+ Schema schema, Runnable onClose) {
super(arrowFileReader, fields);
- this.allocator = new RootAllocator(Long.MAX_VALUE);
- this.filter = filter;
+ this.conditions = toConditionTokens(conditions);
+ this.schema = schema;
this.onClose = onClose;
+ this.filterVectors = new ArrayList<>(schema.getFields().size());
+ this.likePatterns = new HashMap<>();
}
- @Override void evaluateOperator(ArrowRecordBatch arrowRecordBatch) {
+ @Override protected void loadNextArrowBatch() {
+ super.loadNextArrowBatch();
+ final VectorSchemaRoot root;
try {
- this.buf = this.allocator.buffer((long) rowCount * 2);
- this.selectionVector = new SelectionVectorInt16(buf);
- filter.evaluate(arrowRecordBatch, selectionVector);
- } catch (GandivaException e) {
+ root = arrowFileReader.getVectorSchemaRoot();
+ } catch (IOException e) {
throw Util.toUnchecked(e);
}
+ filterVectors.clear();
+ for (int i = 0; i < schema.getFields().size(); i++) {
+ filterVectors.add(root.getVector(i));
+ }
}
@Override public boolean moveNext() {
- if (selectionVector == null
- || selectionVectorIndex >= selectionVector.getRecordCount()) {
- boolean hasNextBatch;
- while (true) {
- try {
- hasNextBatch = arrowFileReader.loadNextBatch();
- } catch (IOException e) {
- throw Util.toUnchecked(e);
+ while (true) {
+ if (currRowIndex >= rowCount - 1) {
+ if (!loadNextNonEmptyArrowBatch()) {
+ return false;
}
- if (hasNextBatch) {
- selectionVectorIndex = 0;
- this.valueVectors.clear();
- loadNextArrowBatch();
- requireNonNull(selectionVector, "selectionVector");
- if (selectionVectorIndex >= selectionVector.getRecordCount()) {
- // the "filtered" batch is empty, but there may be more batches to
fetch
- continue;
- }
- currRowIndex = selectionVector.getIndex(selectionVectorIndex++);
+ }
+ currRowIndex++;
+ if (matches(currRowIndex)) {
+ return true;
+ }
+ }
+ }
+
+ private boolean matches(int rowIndex) {
+ for (List<ConditionToken> orGroup : conditions) {
+ boolean any = false;
+ for (ConditionToken token : orGroup) {
+ if (matches(token, rowIndex)) {
+ any = true;
+ break;
}
- return hasNextBatch;
}
- } else {
- currRowIndex = selectionVector.getIndex(selectionVectorIndex++);
- return true;
+ if (!any) {
+ return false;
+ }
}
+ return true;
}
- @Override public void close() {
- try {
- if (buf != null) {
- buf.close();
+ private boolean matches(ConditionToken token, int rowIndex) {
+ final Object value = getValue(fieldVector(token.fieldName), rowIndex);
+ switch (token.operator) {
+ case IS_NULL:
+ return value == null;
+ case IS_NOT_NULL:
+ return value != null;
+ case IS_TRUE:
+ return Boolean.TRUE.equals(value);
+ case IS_FALSE:
+ return Boolean.FALSE.equals(value);
+ case IS_NOT_TRUE:
+ return !Boolean.TRUE.equals(value);
+ case IS_NOT_FALSE:
+ return !Boolean.FALSE.equals(value);
+ case EQUAL:
+ return value != null && compare(value, literal(token)) == 0;
+ case NOT_EQUAL:
+ return value != null && compare(value, literal(token)) != 0;
+ case LESS_THAN:
+ return value != null && compare(value, literal(token)) < 0;
+ case LESS_THAN_OR_EQUAL:
+ return value != null && compare(value, literal(token)) <= 0;
+ case GREATER_THAN:
+ return value != null && compare(value, literal(token)) > 0;
+ case GREATER_THAN_OR_EQUAL:
+ return value != null && compare(value, literal(token)) >= 0;
+ case LIKE:
+ return value != null
+ && like(value.toString(), requireNonNull(token.value, "value"));
+ default:
+ throw new AssertionError("Unhandled Arrow filter operator: " +
token.operator);
+ }
+ }
+
+ private ValueVector fieldVector(String fieldName) {
+ final Field field = schema.findField(fieldName);
+ final int index = schema.getFields().indexOf(field);
+ if (index < 0) {
+ throw new IllegalArgumentException("Unknown Arrow field: " + fieldName);
+ }
+ return filterVectors.get(index);
+ }
+
+ private static Object literal(ConditionToken token) {
+ final String type = requireNonNull(token.valueType, "valueType");
+ final String value = requireNonNull(token.value, "value");
+ if (type.startsWith("decimal")) {
+ return new BigDecimal(value);
+ } else if (type.equals("integer")) {
+ return Integer.valueOf(value);
+ } else if (type.equals("long")) {
+ return Long.valueOf(value);
+ } else if (type.equals("float")) {
+ return Float.valueOf(value);
+ } else if (type.equals("double")) {
+ return Double.valueOf(value);
+ } else if (type.equals("string")) {
+ return unquote(value);
+ }
+ throw new UnsupportedOperationException("Unsupported literal type: " +
type);
+ }
+
+ private static int compare(Object left, Object right) {
+ if (left instanceof BigDecimal || right instanceof BigDecimal) {
+ return toBigDecimal(left).compareTo(toBigDecimal(right));
+ }
+ if (left instanceof Number && right instanceof Number) {
+ return Double.compare(((Number) left).doubleValue(),
+ ((Number) right).doubleValue());
+ }
+ return left.toString().compareTo(right.toString());
+ }
+
+ private static BigDecimal toBigDecimal(Object value) {
+ if (value instanceof BigDecimal) {
+ return (BigDecimal) value;
+ }
+ return new BigDecimal(value.toString());
+ }
+
+ private boolean like(String value, String pattern) {
+ final String unquotedPattern = unquote(pattern);
+ final Pattern compiledPattern =
+ likePatterns.computeIfAbsent(unquotedPattern, p -> {
+ return Pattern.compile(toRegex(p), Pattern.DOTALL);
+ });
+ return compiledPattern.matcher(value).matches();
+ }
+
+ private static String toRegex(String pattern) {
+ final StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < pattern.length(); i++) {
+ final char c = pattern.charAt(i);
+ if (c == '%') {
+ builder.append(".*");
+ } else if (c == '_') {
+ builder.append('.');
+ } else {
+ builder.append(Pattern.quote(String.valueOf(c)));
}
- filter.close();
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- } finally {
- onClose.run();
}
+ return builder.toString();
+ }
+
+ private static String unquote(String value) {
+ if (value.length() >= 2 && value.charAt(0) == '\''
+ && value.charAt(value.length() - 1) == '\'') {
+ return value.substring(1, value.length() - 1).replace("''", "'");
+ }
+ return value;
+ }
+
+ private static List<List<ConditionToken>> toConditionTokens(
+ List<List<List<String>>> conditions) {
+ final List<List<ConditionToken>> result =
+ new ArrayList<>(conditions.size());
+ for (List<List<String>> orGroup : conditions) {
+ final List<ConditionToken> tokens = new ArrayList<>(orGroup.size());
+ for (List<String> token : orGroup) {
+ tokens.add(ConditionToken.fromTokenList(token));
+ }
+ result.add(tokens);
+ }
+ return result;
+ }
+
+ @Override public void close() {
+ onClose.run();
}
}
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java
deleted file mode 100644
index 0895f36cf1..0000000000
---
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.adapter.arrow;
-
-import org.apache.calcite.util.ImmutableIntList;
-import org.apache.calcite.util.Util;
-
-import org.apache.arrow.gandiva.evaluator.Projector;
-import org.apache.arrow.gandiva.exceptions.GandivaException;
-import org.apache.arrow.vector.ipc.ArrowFileReader;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-
-import java.io.IOException;
-
-/**
- * Enumerator that reads from a projected collection of Arrow value-vectors.
- */
-class ArrowProjectEnumerator extends AbstractArrowEnumerator {
- private final Projector projector;
- private final Runnable onClose;
-
- ArrowProjectEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList
fields,
- Projector projector, Runnable onClose) {
- super(arrowFileReader, fields);
- this.projector = projector;
- this.onClose = onClose;
- }
-
- @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch)
{
- try {
- projector.evaluate(arrowRecordBatch, valueVectors);
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- }
- }
-
- @Override public boolean moveNext() {
- if (currRowIndex >= rowCount - 1) {
- final boolean hasNextBatch;
- try {
- hasNextBatch = arrowFileReader.loadNextBatch();
- } catch (IOException e) {
- throw Util.toUnchecked(e);
- }
- if (hasNextBatch) {
- currRowIndex = 0;
- this.valueVectors.clear();
- loadNextArrowBatch();
- }
- return hasNextBatch;
- } else {
- currRowIndex++;
- return true;
- }
- }
-
- @Override public void close() {
- try {
- projector.close();
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- } finally {
- onClose.run();
- }
- }
-}
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
index 6e268d6469..3da1527f10 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
@@ -100,7 +100,7 @@ RelNode convert(Filter filter) {
final RelTraitSet traitSet =
filter.getTraitSet().replace(ArrowRel.CONVENTION);
// Expand SEARCH (e.g. IN, BETWEEN) before pushing to Arrow,
- // since Gandiva does not support SEARCH natively.
+ // since the Arrow adapter does not support SEARCH natively.
final RexNode condition =
RexUtil.expandSearch(filter.getCluster().getRexBuilder(), null,
filter.getCondition());
return new ArrowFilter(filter.getCluster(), traitSet,
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
index 74438efe2a..2585f5d156 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
@@ -38,17 +38,9 @@
import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.Util;
-import org.apache.arrow.gandiva.evaluator.Filter;
-import org.apache.arrow.gandiva.evaluator.Projector;
-import org.apache.arrow.gandiva.exceptions.GandivaException;
-import org.apache.arrow.gandiva.expression.Condition;
-import org.apache.arrow.gandiva.expression.ExpressionTree;
-import org.apache.arrow.gandiva.expression.TreeBuilder;
-import org.apache.arrow.gandiva.expression.TreeNode;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowFileReader;
import org.apache.arrow.vector.ipc.SeekableReadChannel;
-import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
@@ -58,20 +50,15 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.reflect.Type;
-import java.util.ArrayList;
import java.util.List;
-import static java.lang.Double.parseDouble;
-import static java.lang.Float.parseFloat;
-import static java.lang.Integer.parseInt;
-import static java.lang.Long.parseLong;
import static java.util.Objects.requireNonNull;
/**
* Table backed by an Apache Arrow file.
*
- * <p>Reads data from an Arrow IPC file on disk and supports projection
- * and filter push-down via the Gandiva expression compiler.
+ * <p>Reads data from an Arrow IPC file on disk. Projections and filters read
+ * directly from Arrow value-vectors.
*
* <p>Implements {@link TranslatableTable} so that it can be converted into
* an {@link ArrowTableScan} for query planning, and {@link QueryableTable}
@@ -116,43 +103,6 @@ public class ArrowTable extends AbstractTable
public Enumerable<Object> query(DataContext root, ImmutableIntList fields,
List<List<List<String>>> conditions) {
requireNonNull(fields, "fields");
- final Projector projector;
- final Filter filter;
-
- if (conditions.isEmpty()) {
- filter = null;
- projector = makeProjector(fields);
- } else {
- projector = null;
-
- final List<TreeNode> conjuncts = new ArrayList<>(conditions.size());
- for (List<List<String>> orGroup : conditions) {
- final List<TreeNode> disjuncts = new ArrayList<>(orGroup.size());
- for (List<String> conditionParts : orGroup) {
- disjuncts.add(
- convertConditionToGandiva(
- ConditionToken.fromTokenList(conditionParts)));
- }
- if (disjuncts.size() == 1) {
- conjuncts.add(disjuncts.get(0));
- } else {
- conjuncts.add(TreeBuilder.makeOr(disjuncts));
- }
- }
- final Condition filterCondition;
- if (conjuncts.size() == 1) {
- filterCondition = TreeBuilder.makeCondition(conjuncts.get(0));
- } else {
- filterCondition =
- TreeBuilder.makeCondition(TreeBuilder.makeAnd(conjuncts));
- }
-
- try {
- filter = Filter.make(schema, filterCondition);
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- }
- }
FileInputStream fis = null;
try {
@@ -163,7 +113,7 @@ public Enumerable<Object> query(DataContext root,
ImmutableIntList fields,
final FileInputStream fisRef = fis;
final Runnable onClose = () -> closeSilently(fisRef);
fis = null; // ownership transferred to onClose
- return new ArrowEnumerable(reader, fields, projector, filter, onClose);
+ return new ArrowEnumerable(reader, fields, conditions, schema, onClose);
} catch (IOException e) {
throw Util.toUnchecked(e);
} finally {
@@ -202,70 +152,6 @@ private static RelDataType deduceRowType(Schema schema,
return builder.build();
}
- private @Nullable Projector makeProjector(ImmutableIntList fields) {
- if (requiresDirectVectorProjection(fields)) {
- // Returning null selects ArrowEnumerable's direct vector-read path.
- // Use that path because Gandiva does not support identity projection
- // expressions over Arrow List and binary vectors.
- return null;
- }
-
- final List<ExpressionTree> expressionTrees = new ArrayList<>();
- for (int fieldOrdinal : fields) {
- Field field = schema.getFields().get(fieldOrdinal);
- TreeNode node = TreeBuilder.makeField(field);
- expressionTrees.add(TreeBuilder.makeExpression(node, field));
- }
- try {
- return Projector.make(schema, expressionTrees);
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- }
- }
-
- /** Returns whether selected fields should be projected by reading Arrow
- * value-vectors directly rather than by creating a Gandiva projector.
- *
- * <p>CALCITE-7541 extends this direct projection path for Arrow binary
vector
- * families because Gandiva cannot project them through the existing identity
- * projection path. Queries with filters still use Gandiva filters; this
direct
- * path only applies to no-filter projections.
- */
- private boolean requiresDirectVectorProjection(ImmutableIntList fields) {
- for (int fieldOrdinal : fields) {
- switch (schema.getFields().get(fieldOrdinal).getType().getTypeID()) {
- case List:
- case Binary:
- case LargeBinary:
- case FixedSizeBinary:
- return true;
- default:
- break;
- }
- }
- return false;
- }
-
- /** Converts a single {@link ConditionToken} into a Gandiva {@link
TreeNode}. */
- private TreeNode convertConditionToGandiva(ConditionToken token) {
- final List<TreeNode> treeNodes = new ArrayList<>(2);
- treeNodes.add(
- TreeBuilder.makeField(schema.getFields()
- .get(
- schema.getFields().indexOf(
- schema.findField(token.fieldName)))));
-
- if (token.isBinary()) {
- treeNodes.add(
- makeLiteralNode(
- requireNonNull(token.value, "value"),
- requireNonNull(token.valueType, "valueType")));
- }
-
- return TreeBuilder.makeFunction(
- token.operator, treeNodes, new ArrowType.Bool());
- }
-
/** Closes an {@link AutoCloseable} without throwing. */
private static void closeSilently(AutoCloseable closeable) {
try {
@@ -275,29 +161,6 @@ private static void closeSilently(AutoCloseable closeable)
{
}
}
- private static TreeNode makeLiteralNode(String literal, String type) {
- if (type.startsWith("decimal")) {
- String[] typeParts =
- type.substring(type.indexOf('(') + 1, type.indexOf(')')).split(",");
- int precision = parseInt(typeParts[0]);
- int scale = parseInt(typeParts[1]);
- return TreeBuilder.makeDecimalLiteral(literal, precision, scale);
- } else if (type.equals("integer")) {
- return TreeBuilder.makeLiteral(parseInt(literal));
- } else if (type.equals("long")) {
- return TreeBuilder.makeLiteral(parseLong(literal));
- } else if (type.equals("float")) {
- return TreeBuilder.makeLiteral(parseFloat(literal));
- } else if (type.equals("double")) {
- return TreeBuilder.makeLiteral(parseDouble(literal));
- } else if (type.equals("string")) {
- return TreeBuilder.makeStringLiteral(literal.substring(1,
literal.length() - 1));
- } else {
- throw new IllegalArgumentException("Invalid literal " + literal
- + ", type " + type);
- }
- }
-
/**
* Implementation of {@link Queryable} based on a {@link ArrowTable}.
*
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
index 0ec6804052..2b42295981 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
@@ -35,13 +35,26 @@
import java.util.ArrayList;
import java.util.List;
+import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.EQUAL;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN_OR_EQUAL;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_FALSE;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_FALSE;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_NULL;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_TRUE;
+import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NULL;
+import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_TRUE;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN_OR_EQUAL;
+import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LIKE;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.NOT_EQUAL;
import static
org.apache.calcite.util.DateTimeStringUtils.ISO_DATETIME_FRACTIONAL_SECOND_FORMAT;
import static org.apache.calcite.util.DateTimeStringUtils.getDateFormatter;
import static java.util.Objects.requireNonNull;
/**
- * Translates a {@link RexNode} expression to Gandiva predicate tokens.
+ * Translates a {@link RexNode} expression to Arrow predicate tokens.
*/
class ArrowTranslator {
final RexBuilder rexBuilder;
@@ -65,7 +78,7 @@ public static ArrowTranslator create(RexBuilder rexBuilder,
*
* <p>If exceeded, {@link RexUtil#toCnf(RexBuilder, int, RexNode)} returns
* the original expression unchanged, which may cause the subsequent
- * translation to Gandiva predicates to fail with an
+ * translation to Arrow predicates to fail with an
* {@link UnsupportedOperationException}. When invoked by the Arrow adapter
* module, the exception is caught and the plan falls back to
* an Enumerable convention. */
@@ -120,32 +133,32 @@ private static Object literalValue(RexLiteral literal) {
private ConditionToken translateMatch2(RexNode node) {
switch (node.getKind()) {
case EQUALS:
- return translateBinary("equal", "=", (RexCall) node);
+ return translateBinary(EQUAL, EQUAL, (RexCall) node);
case NOT_EQUALS:
- return translateBinary("not_equal", "<>", (RexCall) node);
+ return translateBinary(NOT_EQUAL, NOT_EQUAL, (RexCall) node);
case LESS_THAN:
- return translateBinary("less_than", ">", (RexCall) node);
+ return translateBinary(LESS_THAN, GREATER_THAN, (RexCall) node);
case LESS_THAN_OR_EQUAL:
- return translateBinary("less_than_or_equal_to", ">=", (RexCall) node);
+ return translateBinary(LESS_THAN_OR_EQUAL, GREATER_THAN_OR_EQUAL,
(RexCall) node);
case GREATER_THAN:
- return translateBinary("greater_than", "<", (RexCall) node);
+ return translateBinary(GREATER_THAN, LESS_THAN, (RexCall) node);
case GREATER_THAN_OR_EQUAL:
- return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node);
+ return translateBinary(GREATER_THAN_OR_EQUAL, LESS_THAN_OR_EQUAL,
(RexCall) node);
case IS_NULL:
- return translateUnary("isnull", (RexCall) node);
+ return translateUnary(IS_NULL, (RexCall) node);
case IS_NOT_NULL:
- return translateUnary("isnotnull", (RexCall) node);
+ return translateUnary(IS_NOT_NULL, (RexCall) node);
case IS_NOT_TRUE:
- return translateUnary("isnottrue", (RexCall) node);
+ return translateUnary(IS_NOT_TRUE, (RexCall) node);
case IS_NOT_FALSE:
- return translateUnary("isnotfalse", (RexCall) node);
+ return translateUnary(IS_NOT_FALSE, (RexCall) node);
case INPUT_REF:
final RexInputRef inputRef = (RexInputRef) node;
- return ConditionToken.unary(fieldNames.get(inputRef.getIndex()),
"istrue");
+ return ConditionToken.unary(fieldNames.get(inputRef.getIndex()),
IS_TRUE);
case NOT:
- return translateUnary("isfalse", (RexCall) node);
+ return translateUnary(IS_FALSE, (RexCall) node);
case LIKE:
- return translateBinary("like", null, (RexCall) node);
+ return translateBinaryNoReverse(LIKE, (RexCall) node);
default:
throw new UnsupportedOperationException("Unsupported operator " + node);
}
@@ -155,7 +168,8 @@ private ConditionToken translateMatch2(RexNode node) {
* Translates a call to a binary operator, reversing arguments if
* necessary.
*/
- private ConditionToken translateBinary(String op, String rop, RexCall call) {
+ private ConditionToken translateBinary(ConditionToken.Operator op,
+ ConditionToken.Operator rop, RexCall call) {
final RexNode left = call.operands.get(0);
final RexNode right = call.operands.get(1);
@Nullable ConditionToken expression = translateBinary2(op, left, right);
@@ -169,9 +183,21 @@ private ConditionToken translateBinary(String op, String
rop, RexCall call) {
throw new UnsupportedOperationException("Unsupported binary operator " +
call);
}
+ /** Translates a call to a binary operator without reversing arguments. */
+ private ConditionToken translateBinaryNoReverse(ConditionToken.Operator op,
+ RexCall call) {
+ final RexNode left = call.operands.get(0);
+ final RexNode right = call.operands.get(1);
+ @Nullable ConditionToken expression = translateBinary2(op, left, right);
+ if (expression != null) {
+ return expression;
+ }
+ throw new UnsupportedOperationException("Unsupported binary operator " +
call);
+ }
+
/** Translates a call to a binary operator. Returns null on failure. */
- private @Nullable ConditionToken translateBinary2(String op, RexNode left,
- RexNode right) {
+ private @Nullable ConditionToken translateBinary2(
+ ConditionToken.Operator op, RexNode left, RexNode right) {
if (right.getKind() != SqlKind.LITERAL) {
return null;
}
@@ -191,7 +217,7 @@ private ConditionToken translateBinary(String op, String
rop, RexCall call) {
/** Combines a field name, operator, and literal to produce a binary
* condition token. */
- private ConditionToken translateOp2(String op, String name,
+ private ConditionToken translateOp2(ConditionToken.Operator op, String name,
RexLiteral right) {
Object value = literalValue(right);
String valueString = value.toString();
@@ -209,7 +235,7 @@ private ConditionToken translateOp2(String op, String name,
}
/** Translates a call to a unary operator. */
- private ConditionToken translateUnary(String op, RexCall call) {
+ private ConditionToken translateUnary(ConditionToken.Operator op, RexCall
call) {
final RexNode opNode = call.operands.get(0);
@Nullable ConditionToken expression = translateUnary2(op, opNode);
@@ -221,7 +247,8 @@ private ConditionToken translateUnary(String op, RexCall
call) {
}
/** Translates a call to a unary operator. Returns null on failure. */
- private @Nullable ConditionToken translateUnary2(String op, RexNode opNode) {
+ private @Nullable ConditionToken translateUnary2(ConditionToken.Operator op,
+ RexNode opNode) {
if (opNode.getKind() == SqlKind.INPUT_REF) {
final RexInputRef inputRef = (RexInputRef) opNode;
final String name = fieldNames.get(inputRef.getIndex());
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java
index 44d3facea7..c5b690840a 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java
@@ -25,7 +25,7 @@
import static java.util.Objects.requireNonNull;
/**
- * A structured representation of a single Gandiva predicate condition.
+ * A structured representation of a single Arrow predicate condition.
*
* <p>A condition is either unary (e.g. {@code IS NULL}) or binary
* (e.g. {@code =}, {@code <}). Unary conditions have a field name
@@ -36,11 +36,11 @@
*/
class ConditionToken {
final String fieldName;
- final String operator;
+ final Operator operator;
final @Nullable String value;
final @Nullable String valueType;
- private ConditionToken(String fieldName, String operator,
+ private ConditionToken(String fieldName, Operator operator,
@Nullable String value, @Nullable String valueType) {
this.fieldName = requireNonNull(fieldName, "fieldName");
this.operator = requireNonNull(operator, "operator");
@@ -50,7 +50,7 @@ private ConditionToken(String fieldName, String operator,
/** Creates a binary condition token
* (e.g. {@code intField equal 12 integer}). */
- static ConditionToken binary(String fieldName, String operator,
+ static ConditionToken binary(String fieldName, Operator operator,
String value, String valueType) {
return new ConditionToken(fieldName, operator,
requireNonNull(value, "value"),
@@ -59,7 +59,7 @@ static ConditionToken binary(String fieldName, String
operator,
/** Creates a unary condition token
* (e.g. {@code intField isnull}). */
- static ConditionToken unary(String fieldName, String operator) {
+ static ConditionToken unary(String fieldName, Operator operator) {
return new ConditionToken(fieldName, operator, null, null);
}
@@ -76,22 +76,55 @@ boolean isBinary() {
* binary conditions. */
List<String> toTokenList() {
if (isBinary()) {
- return ImmutableList.of(fieldName, operator,
+ return ImmutableList.of(fieldName, operator.token,
requireNonNull(value, "value"),
requireNonNull(valueType, "valueType"));
}
- return ImmutableList.of(fieldName, operator);
+ return ImmutableList.of(fieldName, operator.token);
}
/** Creates a {@code ConditionToken} from a serialized string list. */
static ConditionToken fromTokenList(List<String> tokens) {
final int size = tokens.size();
if (size == 4) {
- return binary(tokens.get(0), tokens.get(1),
+ return binary(tokens.get(0), Operator.of(tokens.get(1)),
tokens.get(2), tokens.get(3));
} else if (size == 2) {
- return unary(tokens.get(0), tokens.get(1));
+ return unary(tokens.get(0), Operator.of(tokens.get(1)));
}
throw new IllegalArgumentException("Invalid condition tokens: " + tokens);
}
+
+ /** Operators supported by the Arrow adapter filter representation. */
+ enum Operator {
+ IS_NULL("isnull"),
+ IS_NOT_NULL("isnotnull"),
+ IS_TRUE("istrue"),
+ IS_FALSE("isfalse"),
+ IS_NOT_TRUE("isnottrue"),
+ IS_NOT_FALSE("isnotfalse"),
+ EQUAL("equal"),
+ NOT_EQUAL("not_equal"),
+ LESS_THAN("less_than"),
+ LESS_THAN_OR_EQUAL("less_than_or_equal_to"),
+ GREATER_THAN("greater_than"),
+ GREATER_THAN_OR_EQUAL("greater_than_or_equal_to"),
+ LIKE("like");
+
+ final String token;
+
+ Operator(String token) {
+ this.token = token;
+ }
+
+ static Operator of(String token) {
+ for (Operator operator : values()) {
+ if (operator.token.equals(token)) {
+ return operator;
+ }
+ }
+ throw new UnsupportedOperationException(
+ "Unsupported Arrow filter operator: " + token);
+ }
+ }
}
diff --git
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
index 275bdd0be7..8c9fda7f08 100644
--- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
+++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
@@ -82,6 +82,11 @@ static void initializeArrowState(@TempDir Path sharedTempDir)
arrowDataGenerator.writeArrowData(dataLocationFile);
arrowDataGenerator.writeScottEmpData(arrowFilesDirectory);
+ File emptyBatchDataLocationFile =
+ arrowFilesDirectory.resolve("arrowemptybatch.arrow").toFile();
+ ArrowDataTest emptyBatchDataGenerator = new ArrowDataTest();
+
emptyBatchDataGenerator.writeArrowDataWithEmptyBatch(emptyBatchDataLocationFile);
+
File datatypeLocationFile =
arrowFilesDirectory.resolve("arrowdatatype.arrow").toFile();
ArrowDataTest arrowtypeDataGenerator = new ArrowDataTest();
arrowtypeDataGenerator.writeArrowDataType(datatypeLocationFile);
@@ -260,6 +265,51 @@ static void initializeArrowState(@TempDir Path
sharedTempDir)
.explainContains(plan);
}
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-7580";>[CALCITE-7580]
+ * Remove Gandiva dependency from Arrow adapter</a>. */
+ @Test void testArrowProjectSkipsEmptyBatch() {
+ String sql = "select \"intField\", \"stringField\" from arrowemptybatch\n";
+ String result = "intField=0; stringField=0\n"
+ + "intField=1; stringField=1\n"
+ + "intField=2; stringField=2\n";
+
+ CalciteAssert.that()
+ .with(arrow)
+ .query(sql)
+ .returns(result);
+ }
+
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-7580";>[CALCITE-7580]
+ * Remove Gandiva dependency from Arrow adapter</a>. */
+ @Test void testArrowFilterSkipsEmptyBatch() {
+ String sql = "select \"intField\", \"stringField\"\n"
+ + "from arrowemptybatch\n"
+ + "where \"intField\" > 0";
+ String result = "intField=1; stringField=1\n"
+ + "intField=2; stringField=2\n";
+
+ CalciteAssert.that()
+ .with(arrow)
+ .query(sql)
+ .returns(result);
+ }
+
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-7580";>[CALCITE-7580]
+ * Remove Gandiva dependency from Arrow adapter</a>. */
+ @Test void testArrowFilterSkipsEmptyBatchWithNoMatches() {
+ String sql = "select \"intField\", \"stringField\"\n"
+ + "from arrowemptybatch\n"
+ + "where \"intField\" < 0";
+
+ CalciteAssert.that()
+ .with(arrow)
+ .query(sql)
+ .returns("");
+ }
+
@Test void testArrowProjectFieldsWithIntegerFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
diff --git
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
index a53cc1231a..4b9af441aa 100644
--- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
+++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
@@ -301,6 +301,35 @@ public void writeArrowBinaryData(File file) throws
IOException {
fileOutputStream.close();
}
+ public void writeArrowDataWithEmptyBatch(File file) throws IOException {
+ Schema arrowSchema = makeArrowSchema();
+ try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+ VectorSchemaRoot vectorSchemaRoot =
+ VectorSchemaRoot.create(arrowSchema, allocator);
+ FileOutputStream fileOutputStream = new FileOutputStream(file);
+ ArrowFileWriter arrowFileWriter =
+ new ArrowFileWriter(vectorSchemaRoot, null,
+ fileOutputStream.getChannel())) {
+ arrowFileWriter.start();
+
+ vectorSchemaRoot.setRowCount(0);
+ for (Field field : vectorSchemaRoot.getSchema().getFields()) {
+ vectorSchemaRoot.getVector(field.getName()).setValueCount(0);
+ }
+ arrowFileWriter.writeBatch();
+
+ int rowCount = 3;
+ vectorSchemaRoot.setRowCount(rowCount);
+ intField(vectorSchemaRoot.getVector("intField"), rowCount);
+ varCharField(vectorSchemaRoot.getVector("stringField"), rowCount);
+ floatField(vectorSchemaRoot.getVector("floatField"), rowCount);
+ longField(vectorSchemaRoot.getVector("longField"), rowCount);
+ arrowFileWriter.writeBatch();
+
+ arrowFileWriter.end();
+ }
+ }
+
public void writeArrowDataType(File file) throws IOException {
FileOutputStream fileOutputStream = new FileOutputStream(file);
Schema arrowSchema = makeArrowDateTypeSchema();
diff --git
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java
index ab1f5c2a88..4600dab100 100644
--- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java
+++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java
@@ -18,22 +18,12 @@
import org.apache.calcite.config.CalciteSystemProperty;
-import org.apache.arrow.gandiva.evaluator.Projector;
-import org.apache.arrow.gandiva.exceptions.GandivaException;
-import org.apache.arrow.gandiva.expression.ExpressionTree;
-import org.apache.arrow.vector.types.pojo.Schema;
-
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.ExtensionContext;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* JUnit5 extension to handle Arrow tests.
- *
- * <p>Tests will be skipped if the Gandiva library cannot be loaded on the
given platform.
*/
class ArrowExtension implements ExecutionCondition {
@@ -41,8 +31,7 @@ class ArrowExtension implements ExecutionCondition {
* Whether to run this test.
*
* <p>Enabled by default, unless explicitly disabled from command line
- * ({@code -Dcalcite.test.arrow=false}) or if Gandiva library, used to
implement arrow
- * filtering/projection, cannot be loaded.
+ * ({@code -Dcalcite.test.arrow=false}).
*
* @return {@code true} if the test is enabled and can run in the current
environment,
* {@code false} otherwise
@@ -51,16 +40,6 @@ class ArrowExtension implements ExecutionCondition {
final ExtensionContext context) {
boolean enabled = CalciteSystemProperty.TEST_ARROW.value();
- try {
- Schema emptySchema = new Schema(new ArrayList<>(), null);
- List<ExpressionTree> expressions = new ArrayList<>();
- Projector.make(emptySchema, expressions);
- } catch (GandivaException e) {
- // this exception comes from using an empty expression,
- // but the JNI library was loaded properly
- } catch (UnsatisfiedLinkError e) {
- enabled = false;
- }
if (enabled) {
return ConditionEvaluationResult.enabled("Arrow tests enabled");
diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts
index f00ed7d8e5..64cb532afd 100644
--- a/bom/build.gradle.kts
+++ b/bom/build.gradle.kts
@@ -101,7 +101,6 @@ fun DependencyConstraintHandlerScope.runtimev(
apiv("org.apache.arrow:arrow-memory-netty", "arrow")
apiv("org.apache.arrow:arrow-vector", "arrow")
apiv("org.apache.arrow:arrow-jdbc", "arrow")
- apiv("org.apache.arrow.gandiva:arrow-gandiva", "arrow-gandiva")
apiv("org.apache.calcite.avatica:avatica-core", "calcite.avatica")
apiv("org.apache.calcite.avatica:avatica-server", "calcite.avatica")
apiv("org.apache.cassandra:cassandra-all")
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
index 0ffb60454e..4d7702295b 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
@@ -529,14 +529,11 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs
unknownAs) {
}
if (e.operands.size() == 3 && e.operands.get(2) instanceof RexLiteral) {
final RexLiteral escapeLiteral = (RexLiteral) e.operands.get(2);
- final String escapeStr =
requireNonNull(escapeLiteral.getValueAs(String.class));
- if (escapeStr.length() == 1) {
- char escape = escapeStr.charAt(0);
- e = (RexCall) rexBuilder
- .makeCall(e.getParserPosition(), e.getOperator(),
e.operands.get(0),
- rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape,
'%'),
- e.operands.get(1).getType(), true, true), escapeLiteral);
- }
+ Character escape =
requireNonNull(escapeLiteral.getValueAs(Character.class));
+ e = (RexCall) rexBuilder
+ .makeCall(e.getParserPosition(), e.getOperator(),
e.operands.get(0),
+ rexBuilder.makeLiteral(simplifyLikeString(likeStr, escape,
'%'),
+ e.operands.get(1).getType(), true, true), escapeLiteral);
}
}
return simplifyGenericNode(e);
@@ -546,7 +543,7 @@ private RexNode simplifyLike(RexCall e, RexUnknownAs
unknownAs) {
// string with even escapes 'AA\\\\%%__%%AA' simplify to 'AA\\__%AA'
// string with odd escapes 'AA\\\\\\%%__%%AA' simplify to 'AA\\\\\\%__%AA'
private String simplifyMixedWildcards(String str, char escape) {
- Pattern pattern = getWildCardPattern(escape);
+ Pattern pattern = Pattern.compile("[_%]+");
Matcher matcher = pattern.matcher(str);
StringBuilder builder = new StringBuilder();
int from = 0;
@@ -570,17 +567,6 @@ && consecutiveSameCharCountBefore(str, start - 1, escape)
% 2 == 1) {
return builder.toString();
}
- private static Pattern getWildCardPattern(char escape) {
- switch (escape) {
- case '%':
- return Pattern.compile("_+");
- case '_':
- return Pattern.compile("%+");
- default:
- return Pattern.compile("[_%]+");
- }
- }
-
// Tool method: count the number of consecutive identical characters before
index
private int consecutiveSameCharCountBefore(String str, int index, char
escape) {
int count = 0;
diff --git
a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
index d0ccab7dfd..115b66fa21 100644
--- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
@@ -561,7 +561,7 @@ private static void assertBasic(SqlTypeName typeName) {
}
}
- if (type.getSqlTypeName().getFamily() ==
resultType.getSqlTypeName().getFamily()
+ if (type.getSqlTypeName() == resultType.getSqlTypeName()
&& type.getSqlTypeName().allowsPrec()
&& type.getPrecision() != resultType.getPrecision()) {
final int precision =
diff --git
a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 32017ace1d..4622176a07 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -1870,7 +1870,7 @@ public RelNode convertToSingleValueSubq(
if (leftKeys.size() == 1) {
SqlCall sqlCall =
comparisonOp.createCall(rightVals.getParserPosition(),
leftKeys.get(0), rightVals);
- rexComparison = ensureComparisonTypes(bb.convertExpression(sqlCall));
+ rexComparison = bb.convertExpression(sqlCall);
} else {
assert rightVals instanceof SqlCall;
final SqlBasicCall call = (SqlBasicCall) rightVals;
@@ -1880,10 +1880,9 @@ public RelNode convertToSingleValueSubq(
RexUtil.composeConjunction(rexBuilder,
transform(
Pair.zip(leftKeys, call.getOperandList()),
- pair -> ensureComparisonTypes(
- bb.convertExpression(
-
comparisonOp.createCall(rightVals.getParserPosition(),
- pair.left, pair.right)))));
+ pair -> bb.convertExpression(
+ comparisonOp.createCall(rightVals.getParserPosition(),
+ pair.left, pair.right))));
}
comparisons.add(rexComparison);
}
@@ -1902,31 +1901,6 @@ public RelNode convertToSingleValueSubq(
}
}
- /**
- * Ensures that a comparison expression has matching operand types. If the
- * operands have different type names, casts the right operand to match the
- * left operand's type. This handles the case where type coercion is disabled
- * and the IN-to-OR expansion produces comparisons with mismatched types
- * (e.g., DATE = CHAR).
- */
- private RexNode ensureComparisonTypes(RexNode node) {
- if (validator != null && validator.config().typeCoercionEnabled()) {
- return node;
- }
- if (node instanceof RexCall) {
- final RexCall call = (RexCall) node;
- if (call.operands.size() == 2) {
- final RexNode left = call.operands.get(0);
- final RexNode right = call.operands.get(1);
- if (left.getType().getSqlTypeName() !=
right.getType().getSqlTypeName()) {
- final RexNode castRight = rexBuilder.ensureType(left.getType(),
right, true);
- return rexBuilder.makeCall(call.getOperator(), left, castRight);
- }
- }
- }
- return node;
- }
-
/**
* Converts a {@link SqlNodeList} (for example an IN-list or VALUES list)
* into a relational expression and produces a Rex-level sub-query that
diff --git a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java
b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java
index ac7b5aebe6..2bdc7a1d56 100644
--- a/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java
+++ b/core/src/test/java/org/apache/calcite/rex/RexProgramTest.java
@@ -4358,10 +4358,6 @@ private void checkSarg(String message, Sarg sarg,
* Multiple consecutive '%' in the string matched by LIKE should simplify to
a single '%'</a>,
* <a
href="https://issues.apache.org/jira/browse/CALCITE-7153";>[CALCITE-7153]
* Mixed wildcards of _ and % need to be simplified in LIKE operator</a>.
- * <a
href="https://issues.apache.org/jira/browse/CALCITE-7578";>[CALCITE-7578]
- * LIKE with empty ESCAPE might fail with
StringIndexOutOfBoundsException</a>.
- * <a
href="https://issues.apache.org/jira/browse/CALCITE-7588";>[CALCITE-7588]
- * LIKE with ESCAPE symbols containing wildcards fails</a>.
* */
@Test void testSimplifyLike() {
final RexNode ref = input(tVarchar(true, 10), 0);
@@ -4425,14 +4421,6 @@ private void checkSarg(String message, Sarg sarg,
"LIKE($0, '###%%#%#%A#%%#%A%###%%', '#')");
checkSimplifyUnchanged(like(ref, literal("A"), literal("#")));
checkSimplifyUnchanged(like(ref, literal("%A"), literal("#")));
- checkSimplifyUnchanged(like(ref, literal("TE%_ST"), literal("%")));
- checkSimplifyUnchanged(like(ref, literal("TE%%ST"), literal("%")));
- checkSimplifyUnchanged(like(ref, literal("a%_b%%c"), literal("%")));
- checkSimplifyUnchanged(like(ref, literal("%_%%A%_"), literal("%")));
- // escape char equal to the '_' wildcard. '%E__S%' ESCAPE '_' is a literal
'_'.
- checkSimplifyUnchanged(like(ref, literal("%E__S%"), literal("_")));
- checkSimplifyUnchanged(like(ref, literal("TE_%ST"), literal("_")));
- checkSimplifyUnchanged(like(ref, literal("a_%b__c"), literal("_")));
// As above, but ref is NOT NULL
final RexNode refMandatory = vVarcharNotNull(0);
@@ -4463,14 +4451,6 @@ private void checkSarg(String message, Sarg sarg,
// NOT(SIMILAR TO) is not optimized
checkSimplifyUnchanged(
not(rexBuilder.makeCall(SqlStdOperatorTable.SIMILAR_TO, ref,
literal("%"))));
-
- try {
- // Empty ESCAPE
- checkSimplifyUnchanged(like(ref, literal("a"), literal("")));
- } catch (RuntimeException e) {
- assertThat(e.getMessage(),
- containsString("Invalid escape character ''"));
- }
}
@Test void testSimplifyNullCheckInFilter() {
diff --git
a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java
b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java
index f0ff190d28..8f5e5e4018 100644
--- a/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java
+++ b/core/src/test/java/org/apache/calcite/sql/type/SqlTypeFactoryTest.java
@@ -33,7 +33,6 @@
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasToString;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -173,30 +172,6 @@ class SqlTypeFactoryTest {
assertThat(leastRestrictive.getPrecision(), is(3));
}
- /**
- * Test case for
- * <a href="https://issues.apache.org/jira/browse/CALCITE-7567";>
- * LeastRetrictiveSqlType for TIMESTAMP, TIMESTAMP_LTZ might ignore
precision</a>. */
- @Test void testLeastRestrictiveForTimestampAndTimestampLtz() {
- SqlTypeFixture f = new SqlTypeFixture();
- RelDataType ltz0 =
-
f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0);
- RelDataType leastRestrictive =
- f.typeFactory.leastRestrictive(Lists.newArrayList(ltz0,
f.sqlTimestampPrec3));
- assertThat(leastRestrictive, is(notNullValue()));
- assertThat(leastRestrictive.getPrecision(), is(3));
- }
-
- @Test void testLeastRestrictiveForTimestampLtzAndTimestamp() {
- SqlTypeFixture f = new SqlTypeFixture();
- RelDataType ltz0 =
-
f.typeFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 0);
- RelDataType leastRestrictive =
- f.typeFactory.leastRestrictive(Lists.newArrayList(f.sqlTimestampPrec3,
ltz0));
- assertThat(leastRestrictive, is(notNullValue()));
- assertThat(leastRestrictive.getPrecision(), is(3));
- }
-
@Test void testLeastRestrictiveForTimestampAndDate() {
SqlTypeFixture f = new SqlTypeFixture();
RelDataType leastRestrictive =
diff --git
a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
index 723d749be6..4e8d17cd7a 100644
--- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
+++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
@@ -2213,24 +2213,6 @@ void checkCorrelatedMapSubQuery(boolean expand) {
sql(sql).withExpand(false).ok();
}
- /** Test case for
- * <a
href="https://issues.apache.org/jira/browse/CALCITE-7562";>[CALCITE-7562]
- * SqlToRel misses CAST in case IN expression without type coercion</a>. */
- @Test void testInDateColumnWithoutTypeCoercion() {
- final String sql =
- "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')";
- sql(sql).withTypeCoercion(false).ok();
- }
-
- /** Test case for
- * <a
href="https://issues.apache.org/jira/browse/CALCITE-7562";>[CALCITE-7562]
- * SqlToRel misses CAST in case IN expression without type coercion</a>. */
- @Test void testInDateColumnWithTypeCoercion() {
- final String sql =
- "select * from emp_b where birthdate in ('2000-06-30', '2000-09-27')";
- sql(sql).ok();
- }
-
@Test void testInValueListLong() {
// Go over the default threshold of 20 to force a sub-query.
final String sql = "select empno from emp where deptno in"
diff --git
a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
index 8aa2171d3f..5ee7180efa 100644
--- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
@@ -3266,30 +3266,6 @@ LogicalFilter(condition=[=($cor0.DEPTNO, $7)])
LogicalAggregate(group=[{0}], S=[SUM($1)], agg#1=[COUNT()])
LogicalProject(DEPTNO=[$7], SAL=[$5])
LogicalTableScan(table=[[CATALOG, SALES, EMP]])
-]]>
- </Resource>
- </TestCase>
- <TestCase name="testInDateColumnWithTypeCoercion">
- <Resource name="sql">
- <![CDATA[select * from emp_b where birthdate in ('2000-06-30',
'2000-09-27')]]>
- </Resource>
- <Resource name="plan">
- <![CDATA[
-LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4],
SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], BIRTHDATE=[$9])
- LogicalFilter(condition=[OR(=($9, CAST('2000-06-30'):DATE NOT NULL), =($9,
CAST('2000-09-27'):DATE NOT NULL))])
- LogicalTableScan(table=[[CATALOG, SALES, EMP_B]])
-]]>
- </Resource>
- </TestCase>
- <TestCase name="testInDateColumnWithoutTypeCoercion">
- <Resource name="sql">
- <![CDATA[select * from emp_b where birthdate in ('2000-06-30',
'2000-09-27')]]>
- </Resource>
- <Resource name="plan">
- <![CDATA[
-LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4],
SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], BIRTHDATE=[$9])
- LogicalFilter(condition=[OR(=($9, CAST('2000-06-30'):DATE NOT NULL), =($9,
CAST('2000-09-27'):DATE NOT NULL))])
- LogicalTableScan(table=[[CATALOG, SALES, EMP_B]])
]]>
</Resource>
</TestCase>
diff --git a/gradle.properties b/gradle.properties
index eb0bd778ae..fd82318264 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -81,7 +81,6 @@ jandex.version=3.5.3
# elasticsearch does not like asm:6.2.1+
aggdesigner-algorithm.version=6.1
apiguardian-api.version=1.1.2
-arrow-gandiva.version=15.0.0
arrow.version=15.0.0
asm.version=9.9.1
byte-buddy.version=1.18.8
diff --git a/site/_docs/history.md b/site/_docs/history.md
index aba03c8f3c..ac3be25990 100644
--- a/site/_docs/history.md
+++ b/site/_docs/history.md
@@ -49,6 +49,11 @@ ## <a
href="https://github.com/apache/calcite/releases/tag/calcite-1.43.0";>1.43.
#### Breaking Changes
{: #breaking-1-43-0}
+* [<a
href="https://issues.apache.org/jira/browse/CALCITE-7580";>CALCITE-7580</a>]
+ Remove Gandiva dependency from Arrow adapter. Arrow adapter projection and
+ filter evaluation now run in Java, and the `arrow-gandiva` dependency is no
+ longer included in the Arrow module or BOM.
+
#### New features
{: #new-features-1-43-0}
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java
b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java
index 784cb1eba1..a6685cf952 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlToRelFixture.java
@@ -178,11 +178,6 @@ public SqlToRelFixture withConformance(SqlConformance
conformance) {
.withValidatorConfig(c -> c.withConformance(conformance)));
}
- public SqlToRelFixture withTypeCoercion(boolean enabled) {
- return withFactory(f ->
- f.withValidatorConfig(c -> c.withTypeCoercionEnabled(enabled)));
- }
-
public SqlToRelFixture withDiffRepos(DiffRepository diffRepos) {
return new SqlToRelFixture(sql, decorrelate, tester, factory, trim,
expression, diffRepos);
