This is an automated email from the ASF dual-hosted git repository.
caicancai pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new 410030390a [CALCITE-7580] Remove Gandiva dependency from Arrow adapter
410030390a is described below
commit 410030390a3b97de927d30266c37677bc2896596
Author: Cancai Cai <[email protected]>
AuthorDate: Fri Jun 19 17:23:35 2026 +0800
[CALCITE-7580] Remove Gandiva dependency from Arrow adapter
---
arrow/build.gradle.kts | 1 -
.../adapter/arrow/AbstractArrowEnumerator.java | 30 ++-
.../adapter/arrow/ArrowDirectEnumerator.java | 31 +--
.../calcite/adapter/arrow/ArrowEnumerable.java | 25 +--
.../adapter/arrow/ArrowFilterEnumerator.java | 238 ++++++++++++++++-----
.../adapter/arrow/ArrowProjectEnumerator.java | 80 -------
.../apache/calcite/adapter/arrow/ArrowRules.java | 2 +-
.../apache/calcite/adapter/arrow/ArrowTable.java | 143 +------------
.../calcite/adapter/arrow/ArrowTranslator.java | 69 ++++--
.../calcite/adapter/arrow/ConditionToken.java | 51 ++++-
.../calcite/adapter/arrow/ArrowAdapterTest.java | 50 +++++
.../calcite/adapter/arrow/ArrowDataTest.java | 29 +++
.../calcite/adapter/arrow/ArrowExtension.java | 23 +-
bom/build.gradle.kts | 1 -
gradle.properties | 1 -
site/_docs/history.md | 5 +
16 files changed, 397 insertions(+), 382 deletions(-)
diff --git a/arrow/build.gradle.kts b/arrow/build.gradle.kts
index 598aa8a879..c75a8b5f67 100644
--- a/arrow/build.gradle.kts
+++ b/arrow/build.gradle.kts
@@ -23,7 +23,6 @@
implementation("com.google.guava:guava")
implementation("org.apache.arrow:arrow-memory-netty")
implementation("org.apache.arrow:arrow-vector")
- implementation("org.apache.arrow.gandiva:arrow-gandiva")
annotationProcessor("org.immutables:value")
compileOnly("org.immutables:value-annotations")
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
index e188757b0d..486e3f60bd 100644
---
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
+++
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/AbstractArrowEnumerator.java
@@ -24,9 +24,7 @@
import org.apache.arrow.vector.TimeStampVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
import org.apache.arrow.vector.ipc.ArrowFileReader;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.types.TimeUnit;
import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -51,8 +49,6 @@ abstract class AbstractArrowEnumerator implements
Enumerator<Object> {
this.currRowIndex = -1;
}
- abstract void evaluateOperator(ArrowRecordBatch arrowRecordBatch);
-
protected void loadNextArrowBatch() {
try {
final VectorSchemaRoot vsr = arrowFileReader.getVectorSchemaRoot();
@@ -60,14 +56,32 @@ protected void loadNextArrowBatch() {
this.valueVectors.add(vsr.getVector(i));
}
this.rowCount = vsr.getRowCount();
- VectorUnloader vectorUnloader = new VectorUnloader(vsr);
- ArrowRecordBatch arrowRecordBatch = vectorUnloader.getRecordBatch();
- evaluateOperator(arrowRecordBatch);
} catch (IOException e) {
throw Util.toUnchecked(e);
}
}
+ /** Loads the next non-empty Arrow batch. */
+ protected boolean loadNextNonEmptyArrowBatch() {
+ while (true) {
+ final boolean hasNextBatch;
+ try {
+ hasNextBatch = arrowFileReader.loadNextBatch();
+ } catch (IOException e) {
+ throw Util.toUnchecked(e);
+ }
+ if (!hasNextBatch) {
+ return false;
+ }
+ currRowIndex = -1;
+ valueVectors.clear();
+ loadNextArrowBatch();
+ if (rowCount > 0) {
+ return true;
+ }
+ }
+ }
+
@Override public Object current() {
if (fields.size() == 1) {
return getValue(this.valueVectors.get(0), currRowIndex);
@@ -85,7 +99,7 @@ protected void loadNextArrowBatch() {
* <p>For {@link TimeStampVector}, converts the raw value to
* milliseconds since epoch, which is the representation used by
* Calcite's Enumerable runtime for TIMESTAMP types. */
- private static Object getValue(ValueVector vector, int index) {
+ protected static Object getValue(ValueVector vector, int index) {
if (vector instanceof TimeStampVector) {
if (vector.isNull(index)) {
return null;
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java
index 0cdec7baeb..787ffd88d9 100644
---
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java
+++
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowDirectEnumerator.java
@@ -17,19 +17,11 @@
package org.apache.calcite.adapter.arrow;
import org.apache.calcite.util.ImmutableIntList;
-import org.apache.calcite.util.Util;
import org.apache.arrow.vector.ipc.ArrowFileReader;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-
-import java.io.IOException;
/**
* Enumerator that reads projected Arrow value-vectors directly.
- *
- * <p>This path is used for identity projections that Gandiva cannot project
- * through the existing {@code Projector} path, such as Arrow binary vectors.
- * It is not a replacement for Gandiva expression evaluation.
*/
class ArrowDirectEnumerator extends AbstractArrowEnumerator {
private final Runnable onClose;
@@ -40,27 +32,14 @@ class ArrowDirectEnumerator extends AbstractArrowEnumerator
{
this.onClose = onClose;
}
- @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch)
{
- }
-
@Override public boolean moveNext() {
- if (currRowIndex >= rowCount - 1) {
- final boolean hasNextBatch;
- try {
- hasNextBatch = arrowFileReader.loadNextBatch();
- } catch (IOException e) {
- throw Util.toUnchecked(e);
- }
- if (hasNextBatch) {
- currRowIndex = 0;
- this.valueVectors.clear();
- loadNextArrowBatch();
+ while (currRowIndex >= rowCount - 1) {
+ if (!loadNextNonEmptyArrowBatch()) {
+ return false;
}
- return hasNextBatch;
- } else {
- currRowIndex++;
- return true;
}
+ currRowIndex++;
+ return true;
}
@Override public void close() {
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
index 84ed5997aa..b9c0c4171e 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowEnumerable.java
@@ -21,11 +21,10 @@
import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.Util;
-import org.apache.arrow.gandiva.evaluator.Filter;
-import org.apache.arrow.gandiva.evaluator.Projector;
import org.apache.arrow.vector.ipc.ArrowFileReader;
+import org.apache.arrow.vector.types.pojo.Schema;
-import org.checkerframework.checker.nullness.qual.Nullable;
+import java.util.List;
/**
* Enumerable that reads from Arrow value-vectors.
@@ -33,28 +32,24 @@
class ArrowEnumerable extends AbstractEnumerable<Object> {
private final ArrowFileReader arrowFileReader;
private final ImmutableIntList fields;
- private final @Nullable Projector projector;
- private final @Nullable Filter filter;
+ private final List<List<List<String>>> conditions;
+ private final Schema schema;
private final Runnable onClose;
ArrowEnumerable(ArrowFileReader arrowFileReader, ImmutableIntList fields,
- @Nullable Projector projector, @Nullable Filter filter,
- Runnable onClose) {
+ List<List<List<String>>> conditions, Schema schema, Runnable onClose) {
this.arrowFileReader = arrowFileReader;
- this.projector = projector;
- this.filter = filter;
+ this.conditions = conditions;
+ this.schema = schema;
this.fields = fields;
this.onClose = onClose;
}
@Override public Enumerator<Object> enumerator() {
try {
- if (projector != null) {
- return new ArrowProjectEnumerator(arrowFileReader, fields, projector,
- onClose);
- } else if (filter != null) {
- return new ArrowFilterEnumerator(arrowFileReader, fields, filter,
- onClose);
+ if (!conditions.isEmpty()) {
+ return new ArrowFilterEnumerator(arrowFileReader, fields,
+ conditions, schema, onClose);
}
// No projector and no filter means the query is an identity projection
// that should read selected value-vectors directly.
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java
index 5eddec2249..2f154cc6ef 100644
---
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java
+++
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowFilterEnumerator.java
@@ -19,91 +19,215 @@
import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.Util;
-import org.apache.arrow.gandiva.evaluator.Filter;
-import org.apache.arrow.gandiva.evaluator.SelectionVector;
-import org.apache.arrow.gandiva.evaluator.SelectionVectorInt16;
-import org.apache.arrow.gandiva.exceptions.GandivaException;
-import org.apache.arrow.memory.ArrowBuf;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowFileReader;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-
-import org.checkerframework.checker.nullness.qual.Nullable;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
import static java.util.Objects.requireNonNull;
/**
- * Enumerator that reads from a filtered collection of Arrow value-vectors.
+ * Enumerator that evaluates Arrow filter tokens in Java.
*/
class ArrowFilterEnumerator extends AbstractArrowEnumerator {
- private final BufferAllocator allocator;
- private final Filter filter;
- private @Nullable ArrowBuf buf;
- private @Nullable SelectionVector selectionVector;
- private int selectionVectorIndex;
-
+ private final List<List<ConditionToken>> conditions;
+ private final Schema schema;
private final Runnable onClose;
+ private final List<ValueVector> filterVectors;
+ private final Map<String, Pattern> likePatterns;
- ArrowFilterEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList
fields,
- Filter filter, Runnable onClose) {
+ ArrowFilterEnumerator(ArrowFileReader arrowFileReader,
+ ImmutableIntList fields, List<List<List<String>>> conditions,
+ Schema schema, Runnable onClose) {
super(arrowFileReader, fields);
- this.allocator = new RootAllocator(Long.MAX_VALUE);
- this.filter = filter;
+ this.conditions = toConditionTokens(conditions);
+ this.schema = schema;
this.onClose = onClose;
+ this.filterVectors = new ArrayList<>(schema.getFields().size());
+ this.likePatterns = new HashMap<>();
}
- @Override void evaluateOperator(ArrowRecordBatch arrowRecordBatch) {
+ @Override protected void loadNextArrowBatch() {
+ super.loadNextArrowBatch();
+ final VectorSchemaRoot root;
try {
- this.buf = this.allocator.buffer((long) rowCount * 2);
- this.selectionVector = new SelectionVectorInt16(buf);
- filter.evaluate(arrowRecordBatch, selectionVector);
- } catch (GandivaException e) {
+ root = arrowFileReader.getVectorSchemaRoot();
+ } catch (IOException e) {
throw Util.toUnchecked(e);
}
+ filterVectors.clear();
+ for (int i = 0; i < schema.getFields().size(); i++) {
+ filterVectors.add(root.getVector(i));
+ }
}
@Override public boolean moveNext() {
- if (selectionVector == null
- || selectionVectorIndex >= selectionVector.getRecordCount()) {
- boolean hasNextBatch;
- while (true) {
- try {
- hasNextBatch = arrowFileReader.loadNextBatch();
- } catch (IOException e) {
- throw Util.toUnchecked(e);
+ while (true) {
+ if (currRowIndex >= rowCount - 1) {
+ if (!loadNextNonEmptyArrowBatch()) {
+ return false;
}
- if (hasNextBatch) {
- selectionVectorIndex = 0;
- this.valueVectors.clear();
- loadNextArrowBatch();
- requireNonNull(selectionVector, "selectionVector");
- if (selectionVectorIndex >= selectionVector.getRecordCount()) {
- // the "filtered" batch is empty, but there may be more batches to
fetch
- continue;
- }
- currRowIndex = selectionVector.getIndex(selectionVectorIndex++);
+ }
+ currRowIndex++;
+ if (matches(currRowIndex)) {
+ return true;
+ }
+ }
+ }
+
+ private boolean matches(int rowIndex) {
+ for (List<ConditionToken> orGroup : conditions) {
+ boolean any = false;
+ for (ConditionToken token : orGroup) {
+ if (matches(token, rowIndex)) {
+ any = true;
+ break;
}
- return hasNextBatch;
}
- } else {
- currRowIndex = selectionVector.getIndex(selectionVectorIndex++);
- return true;
+ if (!any) {
+ return false;
+ }
}
+ return true;
}
- @Override public void close() {
- try {
- if (buf != null) {
- buf.close();
+ private boolean matches(ConditionToken token, int rowIndex) {
+ final Object value = getValue(fieldVector(token.fieldName), rowIndex);
+ switch (token.operator) {
+ case IS_NULL:
+ return value == null;
+ case IS_NOT_NULL:
+ return value != null;
+ case IS_TRUE:
+ return Boolean.TRUE.equals(value);
+ case IS_FALSE:
+ return Boolean.FALSE.equals(value);
+ case IS_NOT_TRUE:
+ return !Boolean.TRUE.equals(value);
+ case IS_NOT_FALSE:
+ return !Boolean.FALSE.equals(value);
+ case EQUAL:
+ return value != null && compare(value, literal(token)) == 0;
+ case NOT_EQUAL:
+ return value != null && compare(value, literal(token)) != 0;
+ case LESS_THAN:
+ return value != null && compare(value, literal(token)) < 0;
+ case LESS_THAN_OR_EQUAL:
+ return value != null && compare(value, literal(token)) <= 0;
+ case GREATER_THAN:
+ return value != null && compare(value, literal(token)) > 0;
+ case GREATER_THAN_OR_EQUAL:
+ return value != null && compare(value, literal(token)) >= 0;
+ case LIKE:
+ return value != null
+ && like(value.toString(), requireNonNull(token.value, "value"));
+ default:
+ throw new AssertionError("Unhandled Arrow filter operator: " +
token.operator);
+ }
+ }
+
+ private ValueVector fieldVector(String fieldName) {
+ final Field field = schema.findField(fieldName);
+ final int index = schema.getFields().indexOf(field);
+ if (index < 0) {
+ throw new IllegalArgumentException("Unknown Arrow field: " + fieldName);
+ }
+ return filterVectors.get(index);
+ }
+
+ private static Object literal(ConditionToken token) {
+ final String type = requireNonNull(token.valueType, "valueType");
+ final String value = requireNonNull(token.value, "value");
+ if (type.startsWith("decimal")) {
+ return new BigDecimal(value);
+ } else if (type.equals("integer")) {
+ return Integer.valueOf(value);
+ } else if (type.equals("long")) {
+ return Long.valueOf(value);
+ } else if (type.equals("float")) {
+ return Float.valueOf(value);
+ } else if (type.equals("double")) {
+ return Double.valueOf(value);
+ } else if (type.equals("string")) {
+ return unquote(value);
+ }
+ throw new UnsupportedOperationException("Unsupported literal type: " +
type);
+ }
+
+ private static int compare(Object left, Object right) {
+ if (left instanceof BigDecimal || right instanceof BigDecimal) {
+ return toBigDecimal(left).compareTo(toBigDecimal(right));
+ }
+ if (left instanceof Number && right instanceof Number) {
+ return Double.compare(((Number) left).doubleValue(),
+ ((Number) right).doubleValue());
+ }
+ return left.toString().compareTo(right.toString());
+ }
+
+ private static BigDecimal toBigDecimal(Object value) {
+ if (value instanceof BigDecimal) {
+ return (BigDecimal) value;
+ }
+ return new BigDecimal(value.toString());
+ }
+
+ private boolean like(String value, String pattern) {
+ final String unquotedPattern = unquote(pattern);
+ final Pattern compiledPattern =
+ likePatterns.computeIfAbsent(unquotedPattern, p -> {
+ return Pattern.compile(toRegex(p), Pattern.DOTALL);
+ });
+ return compiledPattern.matcher(value).matches();
+ }
+
+ private static String toRegex(String pattern) {
+ final StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < pattern.length(); i++) {
+ final char c = pattern.charAt(i);
+ if (c == '%') {
+ builder.append(".*");
+ } else if (c == '_') {
+ builder.append('.');
+ } else {
+ builder.append(Pattern.quote(String.valueOf(c)));
}
- filter.close();
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- } finally {
- onClose.run();
}
+ return builder.toString();
+ }
+
+ private static String unquote(String value) {
+ if (value.length() >= 2 && value.charAt(0) == '\''
+ && value.charAt(value.length() - 1) == '\'') {
+ return value.substring(1, value.length() - 1).replace("''", "'");
+ }
+ return value;
+ }
+
+ private static List<List<ConditionToken>> toConditionTokens(
+ List<List<List<String>>> conditions) {
+ final List<List<ConditionToken>> result =
+ new ArrayList<>(conditions.size());
+ for (List<List<String>> orGroup : conditions) {
+ final List<ConditionToken> tokens = new ArrayList<>(orGroup.size());
+ for (List<String> token : orGroup) {
+ tokens.add(ConditionToken.fromTokenList(token));
+ }
+ result.add(tokens);
+ }
+ return result;
+ }
+
+ @Override public void close() {
+ onClose.run();
}
}
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java
deleted file mode 100644
index 0895f36cf1..0000000000
---
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowProjectEnumerator.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.adapter.arrow;
-
-import org.apache.calcite.util.ImmutableIntList;
-import org.apache.calcite.util.Util;
-
-import org.apache.arrow.gandiva.evaluator.Projector;
-import org.apache.arrow.gandiva.exceptions.GandivaException;
-import org.apache.arrow.vector.ipc.ArrowFileReader;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-
-import java.io.IOException;
-
-/**
- * Enumerator that reads from a projected collection of Arrow value-vectors.
- */
-class ArrowProjectEnumerator extends AbstractArrowEnumerator {
- private final Projector projector;
- private final Runnable onClose;
-
- ArrowProjectEnumerator(ArrowFileReader arrowFileReader, ImmutableIntList
fields,
- Projector projector, Runnable onClose) {
- super(arrowFileReader, fields);
- this.projector = projector;
- this.onClose = onClose;
- }
-
- @Override protected void evaluateOperator(ArrowRecordBatch arrowRecordBatch)
{
- try {
- projector.evaluate(arrowRecordBatch, valueVectors);
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- }
- }
-
- @Override public boolean moveNext() {
- if (currRowIndex >= rowCount - 1) {
- final boolean hasNextBatch;
- try {
- hasNextBatch = arrowFileReader.loadNextBatch();
- } catch (IOException e) {
- throw Util.toUnchecked(e);
- }
- if (hasNextBatch) {
- currRowIndex = 0;
- this.valueVectors.clear();
- loadNextArrowBatch();
- }
- return hasNextBatch;
- } else {
- currRowIndex++;
- return true;
- }
- }
-
- @Override public void close() {
- try {
- projector.close();
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- } finally {
- onClose.run();
- }
- }
-}
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
index 6e268d6469..3da1527f10 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
@@ -100,7 +100,7 @@ RelNode convert(Filter filter) {
final RelTraitSet traitSet =
filter.getTraitSet().replace(ArrowRel.CONVENTION);
// Expand SEARCH (e.g. IN, BETWEEN) before pushing to Arrow,
- // since Gandiva does not support SEARCH natively.
+ // since the Arrow adapter does not support SEARCH natively.
final RexNode condition =
RexUtil.expandSearch(filter.getCluster().getRexBuilder(), null,
filter.getCondition());
return new ArrowFilter(filter.getCluster(), traitSet,
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
index 74438efe2a..2585f5d156 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTable.java
@@ -38,17 +38,9 @@
import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.Util;
-import org.apache.arrow.gandiva.evaluator.Filter;
-import org.apache.arrow.gandiva.evaluator.Projector;
-import org.apache.arrow.gandiva.exceptions.GandivaException;
-import org.apache.arrow.gandiva.expression.Condition;
-import org.apache.arrow.gandiva.expression.ExpressionTree;
-import org.apache.arrow.gandiva.expression.TreeBuilder;
-import org.apache.arrow.gandiva.expression.TreeNode;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowFileReader;
import org.apache.arrow.vector.ipc.SeekableReadChannel;
-import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
@@ -58,20 +50,15 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.reflect.Type;
-import java.util.ArrayList;
import java.util.List;
-import static java.lang.Double.parseDouble;
-import static java.lang.Float.parseFloat;
-import static java.lang.Integer.parseInt;
-import static java.lang.Long.parseLong;
import static java.util.Objects.requireNonNull;
/**
* Table backed by an Apache Arrow file.
*
- * <p>Reads data from an Arrow IPC file on disk and supports projection
- * and filter push-down via the Gandiva expression compiler.
+ * <p>Reads data from an Arrow IPC file on disk. Projections and filters read
+ * directly from Arrow value-vectors.
*
* <p>Implements {@link TranslatableTable} so that it can be converted into
* an {@link ArrowTableScan} for query planning, and {@link QueryableTable}
@@ -116,43 +103,6 @@ public class ArrowTable extends AbstractTable
public Enumerable<Object> query(DataContext root, ImmutableIntList fields,
List<List<List<String>>> conditions) {
requireNonNull(fields, "fields");
- final Projector projector;
- final Filter filter;
-
- if (conditions.isEmpty()) {
- filter = null;
- projector = makeProjector(fields);
- } else {
- projector = null;
-
- final List<TreeNode> conjuncts = new ArrayList<>(conditions.size());
- for (List<List<String>> orGroup : conditions) {
- final List<TreeNode> disjuncts = new ArrayList<>(orGroup.size());
- for (List<String> conditionParts : orGroup) {
- disjuncts.add(
- convertConditionToGandiva(
- ConditionToken.fromTokenList(conditionParts)));
- }
- if (disjuncts.size() == 1) {
- conjuncts.add(disjuncts.get(0));
- } else {
- conjuncts.add(TreeBuilder.makeOr(disjuncts));
- }
- }
- final Condition filterCondition;
- if (conjuncts.size() == 1) {
- filterCondition = TreeBuilder.makeCondition(conjuncts.get(0));
- } else {
- filterCondition =
- TreeBuilder.makeCondition(TreeBuilder.makeAnd(conjuncts));
- }
-
- try {
- filter = Filter.make(schema, filterCondition);
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- }
- }
FileInputStream fis = null;
try {
@@ -163,7 +113,7 @@ public Enumerable<Object> query(DataContext root,
ImmutableIntList fields,
final FileInputStream fisRef = fis;
final Runnable onClose = () -> closeSilently(fisRef);
fis = null; // ownership transferred to onClose
- return new ArrowEnumerable(reader, fields, projector, filter, onClose);
+ return new ArrowEnumerable(reader, fields, conditions, schema, onClose);
} catch (IOException e) {
throw Util.toUnchecked(e);
} finally {
@@ -202,70 +152,6 @@ private static RelDataType deduceRowType(Schema schema,
return builder.build();
}
- private @Nullable Projector makeProjector(ImmutableIntList fields) {
- if (requiresDirectVectorProjection(fields)) {
- // Returning null selects ArrowEnumerable's direct vector-read path.
- // Use that path because Gandiva does not support identity projection
- // expressions over Arrow List and binary vectors.
- return null;
- }
-
- final List<ExpressionTree> expressionTrees = new ArrayList<>();
- for (int fieldOrdinal : fields) {
- Field field = schema.getFields().get(fieldOrdinal);
- TreeNode node = TreeBuilder.makeField(field);
- expressionTrees.add(TreeBuilder.makeExpression(node, field));
- }
- try {
- return Projector.make(schema, expressionTrees);
- } catch (GandivaException e) {
- throw Util.toUnchecked(e);
- }
- }
-
- /** Returns whether selected fields should be projected by reading Arrow
- * value-vectors directly rather than by creating a Gandiva projector.
- *
- * <p>CALCITE-7541 extends this direct projection path for Arrow binary
vector
- * families because Gandiva cannot project them through the existing identity
- * projection path. Queries with filters still use Gandiva filters; this
direct
- * path only applies to no-filter projections.
- */
- private boolean requiresDirectVectorProjection(ImmutableIntList fields) {
- for (int fieldOrdinal : fields) {
- switch (schema.getFields().get(fieldOrdinal).getType().getTypeID()) {
- case List:
- case Binary:
- case LargeBinary:
- case FixedSizeBinary:
- return true;
- default:
- break;
- }
- }
- return false;
- }
-
- /** Converts a single {@link ConditionToken} into a Gandiva {@link
TreeNode}. */
- private TreeNode convertConditionToGandiva(ConditionToken token) {
- final List<TreeNode> treeNodes = new ArrayList<>(2);
- treeNodes.add(
- TreeBuilder.makeField(schema.getFields()
- .get(
- schema.getFields().indexOf(
- schema.findField(token.fieldName)))));
-
- if (token.isBinary()) {
- treeNodes.add(
- makeLiteralNode(
- requireNonNull(token.value, "value"),
- requireNonNull(token.valueType, "valueType")));
- }
-
- return TreeBuilder.makeFunction(
- token.operator, treeNodes, new ArrowType.Bool());
- }
-
/** Closes an {@link AutoCloseable} without throwing. */
private static void closeSilently(AutoCloseable closeable) {
try {
@@ -275,29 +161,6 @@ private static void closeSilently(AutoCloseable closeable)
{
}
}
- private static TreeNode makeLiteralNode(String literal, String type) {
- if (type.startsWith("decimal")) {
- String[] typeParts =
- type.substring(type.indexOf('(') + 1, type.indexOf(')')).split(",");
- int precision = parseInt(typeParts[0]);
- int scale = parseInt(typeParts[1]);
- return TreeBuilder.makeDecimalLiteral(literal, precision, scale);
- } else if (type.equals("integer")) {
- return TreeBuilder.makeLiteral(parseInt(literal));
- } else if (type.equals("long")) {
- return TreeBuilder.makeLiteral(parseLong(literal));
- } else if (type.equals("float")) {
- return TreeBuilder.makeLiteral(parseFloat(literal));
- } else if (type.equals("double")) {
- return TreeBuilder.makeLiteral(parseDouble(literal));
- } else if (type.equals("string")) {
- return TreeBuilder.makeStringLiteral(literal.substring(1,
literal.length() - 1));
- } else {
- throw new IllegalArgumentException("Invalid literal " + literal
- + ", type " + type);
- }
- }
-
/**
* Implementation of {@link Queryable} based on a {@link ArrowTable}.
*
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
index 0ec6804052..2b42295981 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
@@ -35,13 +35,26 @@
import java.util.ArrayList;
import java.util.List;
+import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.EQUAL;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.GREATER_THAN_OR_EQUAL;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_FALSE;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_FALSE;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_NULL;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NOT_TRUE;
+import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_NULL;
+import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.IS_TRUE;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.LESS_THAN_OR_EQUAL;
+import static org.apache.calcite.adapter.arrow.ConditionToken.Operator.LIKE;
+import static
org.apache.calcite.adapter.arrow.ConditionToken.Operator.NOT_EQUAL;
import static
org.apache.calcite.util.DateTimeStringUtils.ISO_DATETIME_FRACTIONAL_SECOND_FORMAT;
import static org.apache.calcite.util.DateTimeStringUtils.getDateFormatter;
import static java.util.Objects.requireNonNull;
/**
- * Translates a {@link RexNode} expression to Gandiva predicate tokens.
+ * Translates a {@link RexNode} expression to Arrow predicate tokens.
*/
class ArrowTranslator {
final RexBuilder rexBuilder;
@@ -65,7 +78,7 @@ public static ArrowTranslator create(RexBuilder rexBuilder,
*
* <p>If exceeded, {@link RexUtil#toCnf(RexBuilder, int, RexNode)} returns
* the original expression unchanged, which may cause the subsequent
- * translation to Gandiva predicates to fail with an
+ * translation to Arrow predicates to fail with an
* {@link UnsupportedOperationException}. When invoked by the Arrow adapter
* module, the exception is caught and the plan falls back to
* an Enumerable convention. */
@@ -120,32 +133,32 @@ private static Object literalValue(RexLiteral literal) {
private ConditionToken translateMatch2(RexNode node) {
switch (node.getKind()) {
case EQUALS:
- return translateBinary("equal", "=", (RexCall) node);
+ return translateBinary(EQUAL, EQUAL, (RexCall) node);
case NOT_EQUALS:
- return translateBinary("not_equal", "<>", (RexCall) node);
+ return translateBinary(NOT_EQUAL, NOT_EQUAL, (RexCall) node);
case LESS_THAN:
- return translateBinary("less_than", ">", (RexCall) node);
+ return translateBinary(LESS_THAN, GREATER_THAN, (RexCall) node);
case LESS_THAN_OR_EQUAL:
- return translateBinary("less_than_or_equal_to", ">=", (RexCall) node);
+ return translateBinary(LESS_THAN_OR_EQUAL, GREATER_THAN_OR_EQUAL,
(RexCall) node);
case GREATER_THAN:
- return translateBinary("greater_than", "<", (RexCall) node);
+ return translateBinary(GREATER_THAN, LESS_THAN, (RexCall) node);
case GREATER_THAN_OR_EQUAL:
- return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node);
+ return translateBinary(GREATER_THAN_OR_EQUAL, LESS_THAN_OR_EQUAL,
(RexCall) node);
case IS_NULL:
- return translateUnary("isnull", (RexCall) node);
+ return translateUnary(IS_NULL, (RexCall) node);
case IS_NOT_NULL:
- return translateUnary("isnotnull", (RexCall) node);
+ return translateUnary(IS_NOT_NULL, (RexCall) node);
case IS_NOT_TRUE:
- return translateUnary("isnottrue", (RexCall) node);
+ return translateUnary(IS_NOT_TRUE, (RexCall) node);
case IS_NOT_FALSE:
- return translateUnary("isnotfalse", (RexCall) node);
+ return translateUnary(IS_NOT_FALSE, (RexCall) node);
case INPUT_REF:
final RexInputRef inputRef = (RexInputRef) node;
- return ConditionToken.unary(fieldNames.get(inputRef.getIndex()),
"istrue");
+ return ConditionToken.unary(fieldNames.get(inputRef.getIndex()),
IS_TRUE);
case NOT:
- return translateUnary("isfalse", (RexCall) node);
+ return translateUnary(IS_FALSE, (RexCall) node);
case LIKE:
- return translateBinary("like", null, (RexCall) node);
+ return translateBinaryNoReverse(LIKE, (RexCall) node);
default:
throw new UnsupportedOperationException("Unsupported operator " + node);
}
@@ -155,7 +168,8 @@ private ConditionToken translateMatch2(RexNode node) {
* Translates a call to a binary operator, reversing arguments if
* necessary.
*/
- private ConditionToken translateBinary(String op, String rop, RexCall call) {
+ private ConditionToken translateBinary(ConditionToken.Operator op,
+ ConditionToken.Operator rop, RexCall call) {
final RexNode left = call.operands.get(0);
final RexNode right = call.operands.get(1);
@Nullable ConditionToken expression = translateBinary2(op, left, right);
@@ -169,9 +183,21 @@ private ConditionToken translateBinary(String op, String
rop, RexCall call) {
throw new UnsupportedOperationException("Unsupported binary operator " +
call);
}
+ /** Translates a call to a binary operator without reversing arguments. */
+ private ConditionToken translateBinaryNoReverse(ConditionToken.Operator op,
+ RexCall call) {
+ final RexNode left = call.operands.get(0);
+ final RexNode right = call.operands.get(1);
+ @Nullable ConditionToken expression = translateBinary2(op, left, right);
+ if (expression != null) {
+ return expression;
+ }
+ throw new UnsupportedOperationException("Unsupported binary operator " +
call);
+ }
+
/** Translates a call to a binary operator. Returns null on failure. */
- private @Nullable ConditionToken translateBinary2(String op, RexNode left,
- RexNode right) {
+ private @Nullable ConditionToken translateBinary2(
+ ConditionToken.Operator op, RexNode left, RexNode right) {
if (right.getKind() != SqlKind.LITERAL) {
return null;
}
@@ -191,7 +217,7 @@ private ConditionToken translateBinary(String op, String
rop, RexCall call) {
/** Combines a field name, operator, and literal to produce a binary
* condition token. */
- private ConditionToken translateOp2(String op, String name,
+ private ConditionToken translateOp2(ConditionToken.Operator op, String name,
RexLiteral right) {
Object value = literalValue(right);
String valueString = value.toString();
@@ -209,7 +235,7 @@ private ConditionToken translateOp2(String op, String name,
}
/** Translates a call to a unary operator. */
- private ConditionToken translateUnary(String op, RexCall call) {
+ private ConditionToken translateUnary(ConditionToken.Operator op, RexCall
call) {
final RexNode opNode = call.operands.get(0);
@Nullable ConditionToken expression = translateUnary2(op, opNode);
@@ -221,7 +247,8 @@ private ConditionToken translateUnary(String op, RexCall
call) {
}
/** Translates a call to a unary operator. Returns null on failure. */
- private @Nullable ConditionToken translateUnary2(String op, RexNode opNode) {
+ private @Nullable ConditionToken translateUnary2(ConditionToken.Operator op,
+ RexNode opNode) {
if (opNode.getKind() == SqlKind.INPUT_REF) {
final RexInputRef inputRef = (RexInputRef) opNode;
final String name = fieldNames.get(inputRef.getIndex());
diff --git
a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java
b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java
index 44d3facea7..c5b690840a 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ConditionToken.java
@@ -25,7 +25,7 @@
import static java.util.Objects.requireNonNull;
/**
- * A structured representation of a single Gandiva predicate condition.
+ * A structured representation of a single Arrow predicate condition.
*
* <p>A condition is either unary (e.g. {@code IS NULL}) or binary
* (e.g. {@code =}, {@code <}). Unary conditions have a field name
@@ -36,11 +36,11 @@
*/
class ConditionToken {
final String fieldName;
- final String operator;
+ final Operator operator;
final @Nullable String value;
final @Nullable String valueType;
- private ConditionToken(String fieldName, String operator,
+ private ConditionToken(String fieldName, Operator operator,
@Nullable String value, @Nullable String valueType) {
this.fieldName = requireNonNull(fieldName, "fieldName");
this.operator = requireNonNull(operator, "operator");
@@ -50,7 +50,7 @@ private ConditionToken(String fieldName, String operator,
/** Creates a binary condition token
* (e.g. {@code intField equal 12 integer}). */
- static ConditionToken binary(String fieldName, String operator,
+ static ConditionToken binary(String fieldName, Operator operator,
String value, String valueType) {
return new ConditionToken(fieldName, operator,
requireNonNull(value, "value"),
@@ -59,7 +59,7 @@ static ConditionToken binary(String fieldName, String
operator,
/** Creates a unary condition token
* (e.g. {@code intField isnull}). */
- static ConditionToken unary(String fieldName, String operator) {
+ static ConditionToken unary(String fieldName, Operator operator) {
return new ConditionToken(fieldName, operator, null, null);
}
@@ -76,22 +76,55 @@ boolean isBinary() {
* binary conditions. */
List<String> toTokenList() {
if (isBinary()) {
- return ImmutableList.of(fieldName, operator,
+ return ImmutableList.of(fieldName, operator.token,
requireNonNull(value, "value"),
requireNonNull(valueType, "valueType"));
}
- return ImmutableList.of(fieldName, operator);
+ return ImmutableList.of(fieldName, operator.token);
}
/** Creates a {@code ConditionToken} from a serialized string list. */
static ConditionToken fromTokenList(List<String> tokens) {
final int size = tokens.size();
if (size == 4) {
- return binary(tokens.get(0), tokens.get(1),
+ return binary(tokens.get(0), Operator.of(tokens.get(1)),
tokens.get(2), tokens.get(3));
} else if (size == 2) {
- return unary(tokens.get(0), tokens.get(1));
+ return unary(tokens.get(0), Operator.of(tokens.get(1)));
}
throw new IllegalArgumentException("Invalid condition tokens: " + tokens);
}
+
+ /** Operators supported by the Arrow adapter filter representation. */
+ enum Operator {
+ IS_NULL("isnull"),
+ IS_NOT_NULL("isnotnull"),
+ IS_TRUE("istrue"),
+ IS_FALSE("isfalse"),
+ IS_NOT_TRUE("isnottrue"),
+ IS_NOT_FALSE("isnotfalse"),
+ EQUAL("equal"),
+ NOT_EQUAL("not_equal"),
+ LESS_THAN("less_than"),
+ LESS_THAN_OR_EQUAL("less_than_or_equal_to"),
+ GREATER_THAN("greater_than"),
+ GREATER_THAN_OR_EQUAL("greater_than_or_equal_to"),
+ LIKE("like");
+
+ final String token;
+
+ Operator(String token) {
+ this.token = token;
+ }
+
+ static Operator of(String token) {
+ for (Operator operator : values()) {
+ if (operator.token.equals(token)) {
+ return operator;
+ }
+ }
+ throw new UnsupportedOperationException(
+ "Unsupported Arrow filter operator: " + token);
+ }
+ }
}
diff --git
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
index 275bdd0be7..8c9fda7f08 100644
--- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
+++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
@@ -82,6 +82,11 @@ static void initializeArrowState(@TempDir Path sharedTempDir)
arrowDataGenerator.writeArrowData(dataLocationFile);
arrowDataGenerator.writeScottEmpData(arrowFilesDirectory);
+ File emptyBatchDataLocationFile =
+ arrowFilesDirectory.resolve("arrowemptybatch.arrow").toFile();
+ ArrowDataTest emptyBatchDataGenerator = new ArrowDataTest();
+
emptyBatchDataGenerator.writeArrowDataWithEmptyBatch(emptyBatchDataLocationFile);
+
File datatypeLocationFile =
arrowFilesDirectory.resolve("arrowdatatype.arrow").toFile();
ArrowDataTest arrowtypeDataGenerator = new ArrowDataTest();
arrowtypeDataGenerator.writeArrowDataType(datatypeLocationFile);
@@ -260,6 +265,51 @@ static void initializeArrowState(@TempDir Path
sharedTempDir)
.explainContains(plan);
}
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-7580";>[CALCITE-7580]
+ * Remove Gandiva dependency from Arrow adapter</a>. */
+ @Test void testArrowProjectSkipsEmptyBatch() {
+ String sql = "select \"intField\", \"stringField\" from arrowemptybatch\n";
+ String result = "intField=0; stringField=0\n"
+ + "intField=1; stringField=1\n"
+ + "intField=2; stringField=2\n";
+
+ CalciteAssert.that()
+ .with(arrow)
+ .query(sql)
+ .returns(result);
+ }
+
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-7580";>[CALCITE-7580]
+ * Remove Gandiva dependency from Arrow adapter</a>. */
+ @Test void testArrowFilterSkipsEmptyBatch() {
+ String sql = "select \"intField\", \"stringField\"\n"
+ + "from arrowemptybatch\n"
+ + "where \"intField\" > 0";
+ String result = "intField=1; stringField=1\n"
+ + "intField=2; stringField=2\n";
+
+ CalciteAssert.that()
+ .with(arrow)
+ .query(sql)
+ .returns(result);
+ }
+
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-7580";>[CALCITE-7580]
+ * Remove Gandiva dependency from Arrow adapter</a>. */
+ @Test void testArrowFilterSkipsEmptyBatchWithNoMatches() {
+ String sql = "select \"intField\", \"stringField\"\n"
+ + "from arrowemptybatch\n"
+ + "where \"intField\" < 0";
+
+ CalciteAssert.that()
+ .with(arrow)
+ .query(sql)
+ .returns("");
+ }
+
@Test void testArrowProjectFieldsWithIntegerFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
diff --git
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
index a53cc1231a..4b9af441aa 100644
--- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
+++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowDataTest.java
@@ -301,6 +301,35 @@ public void writeArrowBinaryData(File file) throws
IOException {
fileOutputStream.close();
}
+ public void writeArrowDataWithEmptyBatch(File file) throws IOException {
+ Schema arrowSchema = makeArrowSchema();
+ try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+ VectorSchemaRoot vectorSchemaRoot =
+ VectorSchemaRoot.create(arrowSchema, allocator);
+ FileOutputStream fileOutputStream = new FileOutputStream(file);
+ ArrowFileWriter arrowFileWriter =
+ new ArrowFileWriter(vectorSchemaRoot, null,
+ fileOutputStream.getChannel())) {
+ arrowFileWriter.start();
+
+ vectorSchemaRoot.setRowCount(0);
+ for (Field field : vectorSchemaRoot.getSchema().getFields()) {
+ vectorSchemaRoot.getVector(field.getName()).setValueCount(0);
+ }
+ arrowFileWriter.writeBatch();
+
+ int rowCount = 3;
+ vectorSchemaRoot.setRowCount(rowCount);
+ intField(vectorSchemaRoot.getVector("intField"), rowCount);
+ varCharField(vectorSchemaRoot.getVector("stringField"), rowCount);
+ floatField(vectorSchemaRoot.getVector("floatField"), rowCount);
+ longField(vectorSchemaRoot.getVector("longField"), rowCount);
+ arrowFileWriter.writeBatch();
+
+ arrowFileWriter.end();
+ }
+ }
+
public void writeArrowDataType(File file) throws IOException {
FileOutputStream fileOutputStream = new FileOutputStream(file);
Schema arrowSchema = makeArrowDateTypeSchema();
diff --git
a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java
b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java
index ab1f5c2a88..4600dab100 100644
--- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java
+++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowExtension.java
@@ -18,22 +18,12 @@
import org.apache.calcite.config.CalciteSystemProperty;
-import org.apache.arrow.gandiva.evaluator.Projector;
-import org.apache.arrow.gandiva.exceptions.GandivaException;
-import org.apache.arrow.gandiva.expression.ExpressionTree;
-import org.apache.arrow.vector.types.pojo.Schema;
-
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.ExtensionContext;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* JUnit5 extension to handle Arrow tests.
- *
- * <p>Tests will be skipped if the Gandiva library cannot be loaded on the
given platform.
*/
class ArrowExtension implements ExecutionCondition {
@@ -41,8 +31,7 @@ class ArrowExtension implements ExecutionCondition {
* Whether to run this test.
*
* <p>Enabled by default, unless explicitly disabled from command line
- * ({@code -Dcalcite.test.arrow=false}) or if Gandiva library, used to
implement arrow
- * filtering/projection, cannot be loaded.
+ * ({@code -Dcalcite.test.arrow=false}).
*
* @return {@code true} if the test is enabled and can run in the current
environment,
* {@code false} otherwise
@@ -51,16 +40,6 @@ class ArrowExtension implements ExecutionCondition {
final ExtensionContext context) {
boolean enabled = CalciteSystemProperty.TEST_ARROW.value();
- try {
- Schema emptySchema = new Schema(new ArrayList<>(), null);
- List<ExpressionTree> expressions = new ArrayList<>();
- Projector.make(emptySchema, expressions);
- } catch (GandivaException e) {
- // this exception comes from using an empty expression,
- // but the JNI library was loaded properly
- } catch (UnsatisfiedLinkError e) {
- enabled = false;
- }
if (enabled) {
return ConditionEvaluationResult.enabled("Arrow tests enabled");
diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts
index f00ed7d8e5..64cb532afd 100644
--- a/bom/build.gradle.kts
+++ b/bom/build.gradle.kts
@@ -101,7 +101,6 @@ fun DependencyConstraintHandlerScope.runtimev(
apiv("org.apache.arrow:arrow-memory-netty", "arrow")
apiv("org.apache.arrow:arrow-vector", "arrow")
apiv("org.apache.arrow:arrow-jdbc", "arrow")
- apiv("org.apache.arrow.gandiva:arrow-gandiva", "arrow-gandiva")
apiv("org.apache.calcite.avatica:avatica-core", "calcite.avatica")
apiv("org.apache.calcite.avatica:avatica-server", "calcite.avatica")
apiv("org.apache.cassandra:cassandra-all")
diff --git a/gradle.properties b/gradle.properties
index eb0bd778ae..fd82318264 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -81,7 +81,6 @@ jandex.version=3.5.3
# elasticsearch does not like asm:6.2.1+
aggdesigner-algorithm.version=6.1
apiguardian-api.version=1.1.2
-arrow-gandiva.version=15.0.0
arrow.version=15.0.0
asm.version=9.9.1
byte-buddy.version=1.18.8
diff --git a/site/_docs/history.md b/site/_docs/history.md
index 15e2561d39..c72d6f78d9 100644
--- a/site/_docs/history.md
+++ b/site/_docs/history.md
@@ -49,6 +49,11 @@ ## <a
href="https://github.com/apache/calcite/releases/tag/calcite-1.43.0";>1.43.
#### Breaking Changes
{: #breaking-1-43-0}
+* [<a
href="https://issues.apache.org/jira/browse/CALCITE-7580";>CALCITE-7580</a>]
+ Remove Gandiva dependency from Arrow adapter. Arrow adapter projection and
+ filter evaluation now run in Java, and the `arrow-gandiva` dependency is no
+ longer included in the Arrow module or BOM.
+
#### New features
{: #new-features-1-43-0}
