shizy818 commented on code in PR #14616:
URL: https://github.com/apache/iotdb/pull/14616#discussion_r1909736212
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java:
##########
@@ -65,4 +104,204 @@ public void close() {
public List<IPageReader> loadPageReaderList() {
return this.pageReaderList;
}
+
+ class TsBlockSupplier implements Supplier<TsBlock> {
+ private int[] pageEndOffsets;
+
+ public TsBlockSupplier() {}
+
+ public void setPageEndOffsets(int[] pageEndOffsets) {
+ this.pageEndOffsets = pageEndOffsets;
+ }
+
+ @Override
+ public TsBlock get() {
+ return buildTsBlock();
+ }
+
+ private TsBlock buildTsBlock() {
+ try {
+ List<TSDataType> tsDataTypes = readableChunk.getDataTypes();
+ TsBlockBuilder builder = new TsBlockBuilder(tsDataTypes);
+ writeValidValuesIntoTsBlock(builder);
+ return builder.build();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private boolean isOutOfMemPageBounds() {
+ if (pageEndOffsets == null) {
+ return false;
+ }
+ int[] currTvListOffsets =
timeValuePairIterator.getAlignedTVListOffsets();
+ for (int i = 0; i < pageEndOffsets.length; i++) {
+ if (currTvListOffsets[i] < pageEndOffsets[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void writePageTimeIntoBuilder(long[] time, int count,
TsBlockBuilder builder) {
+ for (int index = 0; index < count; index++) {
+ builder.getTimeColumnBuilder().writeLong(time[index]);
+ }
+ }
+
+ private void writePageValuesIntoBuilder(
+ PageColumnAccessInfo[] columnAccessInfo,
+ List<TSDataType> tsDataTypes,
+ TsBlockBuilder builder) {
+ for (int columnIndex = 0; columnIndex < tsDataTypes.size();
columnIndex++) {
+ PageColumnAccessInfo pageAccessInfo = columnAccessInfo[columnIndex];
+ ColumnBuilder valueBuilder = builder.getColumnBuilder(columnIndex);
+ switch (tsDataTypes.get(columnIndex)) {
+ case BOOLEAN:
+ for (int index = 0; index < pageAccessInfo.count(); index++) {
+ int[] accessInfo = pageAccessInfo.get(index);
+ TsPrimitiveType value =
+ timeValuePairIterator.getPrimitiveObject(accessInfo,
columnIndex);
+ if (value == null) {
+ valueBuilder.appendNull();
+ } else {
+ valueBuilder.writeBoolean(value.getBoolean());
+ }
+ }
+ break;
+ case INT32:
+ case DATE:
+ for (int index = 0; index < pageAccessInfo.count(); index++) {
+ int[] accessInfo = pageAccessInfo.get(index);
+ TsPrimitiveType value =
+ timeValuePairIterator.getPrimitiveObject(accessInfo,
columnIndex);
+ if (value == null) {
+ valueBuilder.appendNull();
+ } else {
+ valueBuilder.writeInt(value.getInt());
+ }
+ }
+ break;
+ case INT64:
+ case TIMESTAMP:
+ for (int index = 0; index < pageAccessInfo.count(); index++) {
+ int[] accessInfo = pageAccessInfo.get(index);
+ TsPrimitiveType value =
+ timeValuePairIterator.getPrimitiveObject(accessInfo,
columnIndex);
+ if (value == null) {
+ valueBuilder.appendNull();
+ } else {
+ valueBuilder.writeLong(value.getLong());
+ }
+ }
+ break;
+ case FLOAT:
+ for (int index = 0; index < pageAccessInfo.count(); index++) {
+ int[] accessInfo = pageAccessInfo.get(index);
+ TsPrimitiveType value =
+ timeValuePairIterator.getPrimitiveObject(accessInfo,
columnIndex);
+ if (value == null) {
+ valueBuilder.appendNull();
+ } else {
+ valueBuilder.writeFloat(value.getFloat());
+ }
+ }
+ break;
+ case DOUBLE:
+ for (int index = 0; index < pageAccessInfo.count(); index++) {
+ int[] accessInfo = pageAccessInfo.get(index);
+ TsPrimitiveType value =
+ timeValuePairIterator.getPrimitiveObject(accessInfo,
columnIndex);
+ if (value == null) {
+ valueBuilder.appendNull();
+ } else {
+ valueBuilder.writeDouble(value.getDouble());
+ }
+ }
+ break;
+ case TEXT:
+ case BLOB:
+ case STRING:
+ for (int index = 0; index < pageAccessInfo.count(); index++) {
+ int[] accessInfo = pageAccessInfo.get(index);
+ TsPrimitiveType value =
+ timeValuePairIterator.getPrimitiveObject(accessInfo,
columnIndex);
+ if (value == null) {
+ valueBuilder.appendNull();
+ } else {
+ valueBuilder.writeBinary(value.getBinary());
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ // read one page and write to tsblock
+ private synchronized void writeValidValuesIntoTsBlock(TsBlockBuilder
builder) {
+ boolean ignoreAllNullRows =
readableChunk.getContext().isIgnoreAllNullRows();
+ List<TSDataType> tsDataTypes = readableChunk.getDataTypes();
+ List<TimeRange> timeColumnDeletion =
readableChunk.getTimeColumnDeletion();
+ List<List<TimeRange>> valueColumnsDeletionList =
readableChunk.getValueColumnsDeletionList();
+
+ int pointsInPage = 0;
+ long[] time = new long[MAX_NUMBER_OF_POINTS_IN_PAGE];
+ PageColumnAccessInfo[] pageColumnAccessInfo = new
PageColumnAccessInfo[tsDataTypes.size()];
Review Comment:
The reason I recalculate pageColumnAccessInfo is for less memory use. In
both `initChunkMetaFromTvLists` and `writeValidValuesIntoTsBlock`, there is one
pageColumnAccessInfo object and it calculates page one by one. If we want to
reuse page information during `initChunkMetaFromTvLists`, I have to remember
all pages until `writeValidValuesIntoTsBlock`. So if it is a boolean aligned
timeseries with 100000 points, it needs 10 pageColumnAccessInfo objects which
are even larger than aligned tvlist itself. How do you think?
I will take a look if I can abstract similar code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
