This is an automated email from the ASF dual-hosted git repository. xiangweiwei pushed a commit to branch readData in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f4574c84e339d6e598c721b2b74f52009a029406 Author: Alima777 <[email protected]> AuthorDate: Sat Nov 27 23:32:37 2021 +0800 read data in batches --- .../iotdb/tsfile/read/TsFileSequenceReader.java | 39 ++++++++++++++-------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 6f5e088..bd82a72 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -67,6 +67,7 @@ public class TsFileSequenceReader implements AutoCloseable { protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); private static final String METADATA_INDEX_NODE_DESERIALIZE_ERROR = "Something error happened while deserializing MetadataIndexNode of file {}"; + private static final int MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024; protected String file; protected TsFileInput tsFileInput; protected long fileMetadataPos; @@ -1000,23 +1001,33 @@ public class TsFileSequenceReader implements AutoCloseable { * * @param position the start position of data in the tsFileInput, or the current position if * position = -1 - * @param size the size of data that want to read + * @param totalSize the size of data that want to read * @return data that been read. */ - protected ByteBuffer readData(long position, int size) throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(size); - if (position < 0) { - if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != size) { - throw new IOException("reach the end of the data"); + protected ByteBuffer readData(long position, int totalSize) throws IOException { + int allocateSize = Math.min(MAX_READ_BUFFER_SIZE, totalSize); + int allocateNum = (int) Math.ceil((double) totalSize / allocateSize); + ByteBuffer buffer = ByteBuffer.allocate(totalSize); + int bufferLimit = 0; + for (int i = 0; i < allocateNum; i++) { + if (i == allocateNum - 1) { + allocateSize = totalSize - allocateSize * (allocateNum - 1); } - } else { - long actualReadSize = ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, size); - if (actualReadSize != size) { - throw new IOException( - String.format( - "reach the end of the data. Size of data that want to read: %s," - + "actual read size: %s, position: %s", - size, actualReadSize, position)); + bufferLimit += allocateSize; + buffer.limit(bufferLimit); + if (position < 0) { + if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != allocateSize) { + throw new IOException("reach the end of the data"); + } + } else { + long actualReadSize = ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, allocateSize); + if (actualReadSize != allocateSize) { + throw new IOException( + String.format( + "reach the end of the data. Size of data that want to read: %s," + + "actual read size: %s, position: %s", + allocateSize, actualReadSize, position)); + } } } buffer.flip();
