This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new ee154606 fix: Reuse previous CometDictionary Java arrays (#489)
ee154606 is described below
commit ee154606e2380a76e7e4f5cbceb7a7b59b33877f
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Tue May 28 13:50:42 2024 -0700
fix: Reuse previous CometDictionary Java arrays (#489)
---
common/src/main/java/org/apache/comet/parquet/ColumnReader.java | 6 +++++-
.../src/main/java/org/apache/comet/vector/CometDictionary.java | 9 ++++++++-
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/common/src/main/java/org/apache/comet/parquet/ColumnReader.java
b/common/src/main/java/org/apache/comet/parquet/ColumnReader.java
index 3d4cb3aa..9e594804 100644
--- a/common/src/main/java/org/apache/comet/parquet/ColumnReader.java
+++ b/common/src/main/java/org/apache/comet/parquet/ColumnReader.java
@@ -234,7 +234,11 @@ public class ColumnReader extends AbstractColumnReader {
Dictionary arrowDictionary =
importer.getProvider().lookup(dictionaryEncoding.getId());
CometPlainVector dictionaryVector =
new CometPlainVector(arrowDictionary.getVector(), useDecimal128,
isUuid);
- dictionary = new CometDictionary(dictionaryVector);
+ if (dictionary != null) {
+ dictionary.setDictionaryVector(dictionaryVector);
+ } else {
+ dictionary = new CometDictionary(dictionaryVector);
+ }
currentVector =
new CometDictionaryVector(
diff --git a/common/src/main/java/org/apache/comet/vector/CometDictionary.java
b/common/src/main/java/org/apache/comet/vector/CometDictionary.java
index 9aa42bd6..b213b75d 100644
--- a/common/src/main/java/org/apache/comet/vector/CometDictionary.java
+++ b/common/src/main/java/org/apache/comet/vector/CometDictionary.java
@@ -26,7 +26,7 @@ import org.apache.spark.unsafe.types.UTF8String;
public class CometDictionary implements AutoCloseable {
private static final int DECIMAL_BYTE_WIDTH = 16;
- private final CometPlainVector values;
+ private CometPlainVector values;
private final int numValues;
/** Decoded dictionary values. Only one of the following is set. */
@@ -47,6 +47,13 @@ public class CometDictionary implements AutoCloseable {
initialize();
}
+ public void setDictionaryVector(CometPlainVector values) {
+ this.values = values;
+ if (values.numValues() != numValues) {
+ throw new IllegalArgumentException("Mismatched dictionary size");
+ }
+ }
+
public ValueVector getValueVector() {
return values.getValueVector();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
