huaxingao commented on code in PR #13949:
URL: https://github.com/apache/iceberg/pull/13949#discussion_r2312046832
##########
arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java:
##########
@@ -217,18 +234,56 @@ public VectorHolder read(VectorHolder reuse, int
numValsToRead) {
}
private void allocateFieldVector(boolean dictionaryEncodedVector) {
+ // Allocate-only: caller must ensure there is no active vector in use.
+ Preconditions.checkState(
+ vec == null,
+ "allocateFieldVector must be called only when no active vector is in
use (vec == null)");
if (dictionaryEncodedVector) {
- allocateDictEncodedVector();
+ // Try to reuse the cached dictionary vector if present
+ if (dictReuseVec != null) {
+ this.vec = dictReuseVec;
+ this.readType = ReadType.DICTIONARY;
+ this.typeWidth = (int) IntVector.TYPE_WIDTH;
+ } else {
+ allocateDictEncodedVector();
+ this.dictReuseVec = this.vec;
+ }
} else {
- Field arrowField =
ArrowSchemaUtil.convert(getPhysicalType(columnDescriptor, icebergField));
- if (columnDescriptor.getPrimitiveType().getLogicalTypeAnnotation() !=
null) {
- allocateVectorBasedOnLogicalType(columnDescriptor.getPrimitiveType(),
arrowField);
+ // Try to reuse the cached plain vector if present
+ if (plainReuseVec != null) {
+ this.vec = plainReuseVec;
+ this.readType = plainReuseReadType;
+ this.typeWidth = plainReuseTypeWidth;
} else {
- allocateVectorBasedOnTypeName(columnDescriptor.getPrimitiveType(),
arrowField);
+ Field arrowField =
ArrowSchemaUtil.convert(getPhysicalType(columnDescriptor, icebergField));
+ if (columnDescriptor.getPrimitiveType().getLogicalTypeAnnotation() !=
null) {
+
allocateVectorBasedOnLogicalType(columnDescriptor.getPrimitiveType(),
arrowField);
+ } else {
+ allocateVectorBasedOnTypeName(columnDescriptor.getPrimitiveType(),
arrowField);
+ }
+ // Cache for reuse next time we switch back to plain
+ this.plainReuseVec = this.vec;
+ this.plainReuseReadType = this.readType;
+ this.plainReuseTypeWidth = this.typeWidth;
}
}
}
+ private void clearReuseCaches() {
+ if (this.dictReuseVec != null && this.dictReuseVec != this.vec) {
+ this.dictReuseVec.close();
+ }
+
+ if (this.plainReuseVec != null && this.plainReuseVec != this.vec) {
+ this.plainReuseVec.close();
+ }
+
+ this.dictReuseVec = null;
Review Comment:
If cached == vec: do not close (still in use), but still null the cache so
we never reuse a soon-to-be-closed vector.
This behavior is exercised by
`TestArrowReader.testReadAllWithSmallerBatchSize`; moving the nulling inside
the if caused an NPE
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
