This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 448046fc add ioSizeRecorder (#631)
448046fc is described below
commit 448046fc1becae92c7af5346014572618a54f342
Author: shuwenwei <[email protected]>
AuthorDate: Wed Nov 26 10:21:40 2025 +0800
add ioSizeRecorder (#631)
---
.../apache/tsfile/read/TsFileDeviceIterator.java | 20 +++++---
.../apache/tsfile/read/TsFileSequenceReader.java | 56 ++++++++++++++--------
2 files changed, 50 insertions(+), 26 deletions(-)
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileDeviceIterator.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileDeviceIterator.java
index 66fd3fbc..fc283f26 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileDeviceIterator.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileDeviceIterator.java
@@ -41,6 +41,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Queue;
+import java.util.function.LongConsumer;
public class TsFileDeviceIterator implements Iterator<Pair<IDeviceID,
Boolean>> {
@@ -49,6 +50,7 @@ public class TsFileDeviceIterator implements
Iterator<Pair<IDeviceID, Boolean>>
private final Iterator<MetadataIndexNode> tableMetadataIndexNodeIterator;
private final Queue<Pair<IDeviceID, long[]>> queue = new LinkedList<>();
private final List<long[]> leafDeviceNodeOffsetList = new LinkedList<>();
+ private final LongConsumer ioSizeRecorder;
private Pair<IDeviceID, Boolean> currentDevice = null;
private MetadataIndexNode measurementNode;
@@ -59,13 +61,17 @@ public class TsFileDeviceIterator implements
Iterator<Pair<IDeviceID, Boolean>>
this.deserializeConfig = reader.getDeserializeContext();
this.tableMetadataIndexNodeIterator =
reader.readFileMetadata().getTableMetadataIndexNodeMap().values().iterator();
+ this.ioSizeRecorder = null;
}
- public TsFileDeviceIterator(TsFileSequenceReader reader, String tableName)
throws IOException {
+ public TsFileDeviceIterator(
+ TsFileSequenceReader reader, String tableName, LongConsumer
ioSizeRecorder)
+ throws IOException {
this.reader = reader;
this.deserializeConfig = reader.getDeserializeContext();
+ this.ioSizeRecorder = ioSizeRecorder;
MetadataIndexNode tableMetadataIndexNode =
- reader.readFileMetadata().getTableMetadataIndexNode(tableName);
+
reader.readFileMetadata(ioSizeRecorder).getTableMetadataIndexNode(tableName);
this.tableMetadataIndexNodeIterator =
tableMetadataIndexNode == null
? Collections.emptyIterator()
@@ -122,7 +128,8 @@ public class TsFileDeviceIterator implements
Iterator<Pair<IDeviceID, Boolean>>
try {
// get the first measurement node of this device, to know if the device
is aligned
this.measurementNode =
- reader.readMetadataIndexNode(startEndPair.right[0],
startEndPair.right[1], false);
+ reader.readMetadataIndexNode(
+ startEndPair.right[0], startEndPair.right[1], false,
ioSizeRecorder);
boolean isAligned = reader.isAlignedDevice(measurementNode);
currentDevice = new Pair<>(startEndPair.left, isAligned);
return currentDevice;
@@ -147,7 +154,7 @@ public class TsFileDeviceIterator implements
Iterator<Pair<IDeviceID, Boolean>>
Long startOffset, Long endOffset, Queue<Pair<IDeviceID, long[]>>
measurementNodeOffsetQueue)
throws IOException {
try {
- ByteBuffer nextBuffer = reader.readData(startOffset, endOffset);
+ ByteBuffer nextBuffer = reader.readData(startOffset, endOffset,
ioSizeRecorder);
MetadataIndexNode deviceLeafNode =
deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
nextBuffer, deserializeConfig);
@@ -210,7 +217,8 @@ public class TsFileDeviceIterator implements
Iterator<Pair<IDeviceID, Boolean>>
// first entry, because the rest are the same
MetadataIndexNodeType nodeType =
MetadataIndexNodeType.deserialize(
- ReadWriteIOUtils.readByte(reader.readData(endOffset - 1,
endOffset)));
+ ReadWriteIOUtils.readByte(
+ reader.readData(endOffset - 1, endOffset,
ioSizeRecorder)));
isCurrentLayerLeafNode =
nodeType.equals(MetadataIndexNodeType.LEAF_DEVICE);
}
if (isCurrentLayerLeafNode) {
@@ -219,7 +227,7 @@ public class TsFileDeviceIterator implements
Iterator<Pair<IDeviceID, Boolean>>
leafDeviceNodeOffsets.add(offset);
continue;
}
- ByteBuffer nextBuffer = reader.readData(startOffset, endOffset);
+ ByteBuffer nextBuffer = reader.readData(startOffset, endOffset,
ioSizeRecorder);
getAllDeviceLeafNodeOffset(
deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
nextBuffer, deserializeConfig),
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index 73b5082a..da8d0169 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -729,7 +729,7 @@ public class TsFileSequenceReader implements AutoCloseable {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
}
- firstTimeseriesMetadata = getTimeColumnMetadata(metadataIndexNode);
+ firstTimeseriesMetadata = getTimeColumnMetadata(metadataIndexNode, null);
metadataIndexPair =
getMetadataAndEndOffsetOfMeasurementNode(
metadataIndexNode, path.getMeasurement(), false, null);
@@ -971,7 +971,8 @@ public class TsFileSequenceReader implements AutoCloseable {
}
// Get the time column metadata if the device is aligned
- TimeseriesMetadata timeColumnMetadata =
getTimeColumnMetadata(measurementMetadataIndexNode);
+ TimeseriesMetadata timeColumnMetadata =
+ getTimeColumnMetadata(measurementMetadataIndexNode, null);
List<TimeseriesMetadata> valueTimeseriesMetadataList =
timeColumnMetadata != null ? new ArrayList<>() : null;
@@ -1123,9 +1124,9 @@ public class TsFileSequenceReader implements
AutoCloseable {
return new TsFileDeviceIterator(this);
}
- public TsFileDeviceIterator getTableDevicesIteratorWithIsAligned(String
tableName)
- throws IOException {
- return new TsFileDeviceIterator(this, tableName);
+ public TsFileDeviceIterator getTableDevicesIteratorWithIsAligned(
+ String tableName, LongConsumer ioSizeRecorder) throws IOException {
+ return new TsFileDeviceIterator(this, tableName, ioSizeRecorder);
}
/**
@@ -1276,35 +1277,42 @@ public class TsFileSequenceReader implements
AutoCloseable {
return "".equals(((MeasurementMetadataIndexEntry) entry).getName());
}
- TimeseriesMetadata getTimeColumnMetadata(MetadataIndexNode measurementNode)
throws IOException {
+ public TimeseriesMetadata getTimeColumnMetadata(
+ MetadataIndexNode rootMeasurementNode, LongConsumer ioSizeRecorder)
throws IOException {
// Not aligned timeseries
- if (!isAlignedDevice(measurementNode)) {
+ if (!isAlignedDevice(rootMeasurementNode)) {
return null;
}
// Aligned timeseries
- if
(measurementNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ if
(rootMeasurementNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT))
{
ByteBuffer buffer;
- if (measurementNode.getChildren().size() > 1) {
+ if (rootMeasurementNode.getChildren().size() > 1) {
buffer =
readData(
- measurementNode.getChildren().get(0).getOffset(),
- measurementNode.getChildren().get(1).getOffset());
+ rootMeasurementNode.getChildren().get(0).getOffset(),
+ rootMeasurementNode.getChildren().get(1).getOffset(),
+ ioSizeRecorder);
} else {
buffer =
readData(
- measurementNode.getChildren().get(0).getOffset(),
measurementNode.getEndOffset());
+ rootMeasurementNode.getChildren().get(0).getOffset(),
+ rootMeasurementNode.getEndOffset(),
+ ioSizeRecorder);
}
return TimeseriesMetadata.deserializeFrom(buffer, true);
- } else if
(measurementNode.getNodeType().equals(MetadataIndexNodeType.INTERNAL_MEASUREMENT))
{
+ } else if (rootMeasurementNode
+ .getNodeType()
+ .equals(MetadataIndexNodeType.INTERNAL_MEASUREMENT)) {
ByteBuffer buffer =
readData(
- measurementNode.getChildren().get(0).getOffset(),
- measurementNode.getChildren().get(1).getOffset());
+ rootMeasurementNode.getChildren().get(0).getOffset(),
+ rootMeasurementNode.getChildren().get(1).getOffset(),
+ ioSizeRecorder);
MetadataIndexNode metadataIndexNode =
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
buffer, deserializeConfig);
- return getTimeColumnMetadata(metadataIndexNode);
+ return getTimeColumnMetadata(metadataIndexNode, ioSizeRecorder);
}
return null;
}
@@ -1389,7 +1397,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
List<TimeseriesMetadata> timeseriesMetadataList,
MetadataIndexNode measurementNode,
Set<String> excludedMeasurementIds,
- boolean needChunkMetadata)
+ boolean needChunkMetadata,
+ LongConsumer ioSizeRecorder)
throws IOException {
int metadataIndexListSize = measurementNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
@@ -1417,7 +1426,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
timeseriesMetadataList,
nextLayerMeasurementNode,
excludedMeasurementIds,
- needChunkMetadata);
+ needChunkMetadata,
+ ioSizeRecorder);
}
}
}
@@ -2773,7 +2783,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
public List<AbstractAlignedChunkMetadata>
getAlignedChunkMetadataByMetadataIndexNode(
IDeviceID device, MetadataIndexNode metadataIndexNode, boolean
ignoreAllNullRows)
throws IOException {
- TimeseriesMetadata firstTimeseriesMetadata =
getTimeColumnMetadata(metadataIndexNode);
+ TimeseriesMetadata firstTimeseriesMetadata =
getTimeColumnMetadata(metadataIndexNode, null);
if (firstTimeseriesMetadata == null) {
throw new IOException("Timeseries of device {" + device + "} are not
aligned");
}
@@ -2951,8 +2961,14 @@ public class TsFileSequenceReader implements
AutoCloseable {
*/
public MetadataIndexNode readMetadataIndexNode(
long startOffset, long endOffset, boolean isDeviceLevel) throws
IOException {
+ return readMetadataIndexNode(startOffset, endOffset, isDeviceLevel, null);
+ }
+
+ public MetadataIndexNode readMetadataIndexNode(
+ long startOffset, long endOffset, boolean isDeviceLevel, LongConsumer
ioSizeRecorder)
+ throws IOException {
MetadataIndexNode metadataIndexNode;
- final ByteBuffer buffer = readData(startOffset, endOffset);
+ final ByteBuffer buffer = readData(startOffset, endOffset, ioSizeRecorder);
if (isDeviceLevel) {
metadataIndexNode =
deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
