pavibhai commented on a change in pull request #1072:
URL: https://github.com/apache/orc/pull/1072#discussion_r839777744
##########
File path: java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java
##########
@@ -584,4 +562,169 @@ public void putBuffer(ByteBuffer buffer) {
} while (tree.putIfAbsent(key, buffer) != null);
}
}
+
+ static class ChunkReader {
+ private final BufferChunk from;
+ private final BufferChunk to;
+ private final int readBytes;
+ private final int reqBytes;
+
+ private ChunkReader(BufferChunk from, BufferChunk to, int readSize, int
reqBytes) {
+ this.from = from;
+ this.to = to;
+ this.readBytes = readSize;
+ this.reqBytes = reqBytes;
+ }
+
+ double getExtraBytesFraction() {
+ return (readBytes - reqBytes) / ((double) reqBytes);
+ }
+
+ public int getReadBytes() {
+ return readBytes;
+ }
+
+ public int getReqBytes() {
+ return reqBytes;
+ }
+
+ public BufferChunk getFrom() {
+ return from;
+ }
+
+ public BufferChunk getTo() {
+ return to;
+ }
+
+ void populateChunks(ByteBuffer bytes, boolean allocateDirect, double
extraByteTolerance) {
+ if (getExtraBytesFraction() > extraByteTolerance) {
+ LOG.debug("ExtraBytesFraction = {}, ExtraByteTolerance = {}, reducing
memory size",
+ getExtraBytesFraction(),
+ extraByteTolerance);
+ populateChunksReduceSize(bytes, allocateDirect);
+ } else {
+ LOG.debug("ExtraBytesFraction = {}, ExtraByteTolerance = {},
populating as is",
+ getExtraBytesFraction(),
+ extraByteTolerance);
+ populateChunksAsIs(bytes);
+ }
+ }
+
+ void populateChunksAsIs(ByteBuffer bytes) {
+ // populate each BufferChunks with the data
+ BufferChunk current = from;
+ long offset = from.getOffset();
+ while (current != to.next) {
+ ByteBuffer currentBytes = current == to ? bytes : bytes.duplicate();
+ currentBytes.position((int) (current.getOffset() - offset));
+ currentBytes.limit((int) (current.getEnd() - offset));
+ current.setChunk(currentBytes);
+ current = (BufferChunk) current.next;
+ }
+ }
+
+ void populateChunksReduceSize(ByteBuffer bytes, boolean allocateDirect) {
+ byte[] newBytes = new byte[reqBytes];
+ ByteBuffer newBuffer;
+ if (allocateDirect) {
+ newBuffer = ByteBuffer.allocateDirect(reqBytes);
+ newBuffer.position(reqBytes);
+ newBuffer.flip();
+ } else {
+ newBuffer = ByteBuffer.wrap(newBytes);
Review comment:
Good catch, fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscr...@orc.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
