This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new e17d9eb1795 [IOTDB-6249] Load: Streaming read timeseries metadata from
tsfile when auto creating or verifying schema (#11540)
e17d9eb1795 is described below
commit e17d9eb1795a8fff9b344323b95a58cbe85cf167
Author: Itami Sho <[email protected]>
AuthorDate: Tue Nov 21 20:31:09 2023 +0800
[IOTDB-6249] Load: Streaming read timeseries metadata from tsfile when auto
creating or verifying schema (#11540)
---
.../plan/analyze/LoadTsfileAnalyzer.java | 225 ++++++++-------------
...eReaderTimeseriesMetadataIteratorException.java | 27 +++
.../file/metadata/enums/MetadataIndexNodeType.java | 2 +-
...leSequenceReaderTimeseriesMetadataIterator.java | 220 ++++++++++++++++++++
...quenceReaderTimeseriesMetadataIteratorTest.java | 62 ++++++
5 files changed, 391 insertions(+), 145 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
index 2ef08186e88..ee1ad09336c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
@@ -34,7 +34,6 @@ import
org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
import org.apache.iotdb.db.auth.AuthorityChecker;
-import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.LoadFileException;
import org.apache.iotdb.db.exception.LoadReadOnlyException;
@@ -68,6 +67,7 @@ import
org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import
org.apache.iotdb.tsfile.read.TsFileSequenceReaderTimeseriesMetadataIterator;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -82,7 +82,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -92,8 +91,6 @@ public class LoadTsfileAnalyzer {
private static final Logger LOGGER =
LoggerFactory.getLogger(LoadTsfileAnalyzer.class);
- private static final IoTDBConfig CONFIG =
IoTDBDescriptor.getInstance().getConfig();
-
private static final IClientManager<ConfigRegionId, ConfigNodeClient>
CONFIG_NODE_CLIENT_MANAGER =
ConfigNodeClientManager.getInstance();
@@ -188,42 +185,50 @@ public class LoadTsfileAnalyzer {
private void analyzeSingleTsFile(File tsFile) throws IOException,
AuthException {
try (final TsFileSequenceReader reader = new
TsFileSequenceReader(tsFile.getAbsolutePath())) {
// can be reused when constructing tsfile resource
- final Map<String, List<TimeseriesMetadata>> device2TimeseriesMetadata =
- reader.getAllTimeseriesMetadata(true);
- if (device2TimeseriesMetadata.isEmpty()) {
- LOGGER.warn("device2TimeseriesMetadata is empty, because maybe the
tsfile is empty");
- return;
- }
-
- // auto create or verify schema
- if (IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled()
- || loadTsFileStatement.isVerifySchema()) {
-
- final TimeSeriesIterator timeSeriesIterator =
- new TimeSeriesIterator(tsFile, device2TimeseriesMetadata);
- while (timeSeriesIterator.hasNext()) {
- schemaAutoCreatorAndVerifier.autoCreateAndVerify(reader,
timeSeriesIterator);
- }
+ final TsFileSequenceReaderTimeseriesMetadataIterator
timeseriesMetadataIterator =
+ new TsFileSequenceReaderTimeseriesMetadataIterator(reader, true);
-
schemaAutoCreatorAndVerifier.flushAndClearDeviceIsAlignedCacheIfNecessary();
- }
+ long writePointCount = 0;
// construct tsfile resource
final TsFileResource tsFileResource = new TsFileResource(tsFile);
if (!tsFileResource.resourceFileExists()) {
// it will be serialized in LoadSingleTsFileNode
- FileLoaderUtils.updateTsFileResource(device2TimeseriesMetadata,
tsFileResource);
tsFileResource.updatePlanIndexes(reader.getMinPlanIndex());
tsFileResource.updatePlanIndexes(reader.getMaxPlanIndex());
} else {
tsFileResource.deserialize();
}
+ // auto create or verify schema
+ if (IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled()
+ || loadTsFileStatement.isVerifySchema()) {
+ // check if the tsfile is empty
+ if (!timeseriesMetadataIterator.hasNext()) {
+ LOGGER.warn("device2TimeseriesMetadata is empty, because maybe the
tsfile is empty");
+ return;
+ }
+
+ while (timeseriesMetadataIterator.hasNext()) {
+ Map<String, List<TimeseriesMetadata>> device2TimeseriesMetadata =
+ timeseriesMetadataIterator.next();
+
+ schemaAutoCreatorAndVerifier.autoCreateAndVerify(reader,
device2TimeseriesMetadata);
+
+ if (!tsFileResource.resourceFileExists()) {
+ FileLoaderUtils.updateTsFileResource(device2TimeseriesMetadata,
tsFileResource);
+ }
+ writePointCount += getWritePointCount(device2TimeseriesMetadata);
+ }
+
+
schemaAutoCreatorAndVerifier.flushAndClearDeviceIsAlignedCacheIfNecessary();
+ }
+
TimestampPrecisionUtils.checkTimestampPrecision(tsFileResource.getFileEndTime());
tsFileResource.setStatus(TsFileResourceStatus.NORMAL);
loadTsFileStatement.addTsFileResource(tsFileResource);
-
loadTsFileStatement.addWritePointCount(getWritePointCount(device2TimeseriesMetadata));
+ loadTsFileStatement.addWritePointCount(writePointCount);
}
}
@@ -234,142 +239,76 @@ public class LoadTsfileAnalyzer {
.sum();
}
- private static final class TimeSeriesIterator
- implements Iterator<Pair<String, TimeseriesMetadata>> {
-
- private static final Logger LOGGER =
LoggerFactory.getLogger(TimeSeriesIterator.class);
-
- private static final long LOG_PRINT_INTERVAL = 10000;
- private long returnedTimeseriesCount = 0;
- private boolean lastLogHasPrinted = false;
-
- private final File tsFile;
- private final Iterator<Map.Entry<String, List<TimeseriesMetadata>>>
- device2TimeseriesMetadataIterator;
-
- private String currentDevice;
- private Iterator<TimeseriesMetadata> timeseriesMetadataIterator;
-
- public TimeSeriesIterator(
- File tsFile, Map<String, List<TimeseriesMetadata>>
device2TimeseriesMetadata) {
- this.tsFile = tsFile;
- this.device2TimeseriesMetadataIterator =
device2TimeseriesMetadata.entrySet().iterator();
- }
-
- @Override
- public boolean hasNext() {
- if (timeseriesMetadataIterator == null ||
!timeseriesMetadataIterator.hasNext()) {
- if (device2TimeseriesMetadataIterator.hasNext()) {
- final Map.Entry<String, List<TimeseriesMetadata>> next =
- device2TimeseriesMetadataIterator.next();
- currentDevice = next.getKey();
- timeseriesMetadataIterator = next.getValue().iterator();
- } else {
- if (!lastLogHasPrinted) {
- LOGGER.info(
- "Analyzing TsFile {}, all {} timeseries has been returned to
analyzer.",
- tsFile.getAbsolutePath(),
- returnedTimeseriesCount);
- lastLogHasPrinted = true;
- }
- return false;
- }
- }
- return timeseriesMetadataIterator.hasNext();
- }
-
- @Override
- public Pair<String, TimeseriesMetadata> next() {
- if (returnedTimeseriesCount == 0) {
- LOGGER.info(
- "Analyzing TsFile {}, start to return timeseries to analyzer.",
- tsFile.getAbsolutePath());
- } else if (returnedTimeseriesCount % LOG_PRINT_INTERVAL == 0) {
- LOGGER.info(
- "Analyzing TsFile {}, until now {} timeseries has been returned to
analyzer.",
- tsFile.getAbsolutePath(),
- returnedTimeseriesCount);
- }
- returnedTimeseriesCount++;
-
- return new Pair<>(currentDevice, timeseriesMetadataIterator.next());
- }
- }
-
private final class SchemaAutoCreatorAndVerifier {
private final Map<String, Boolean> tsfileDevice2IsAligned = new
HashMap<>();
-
- private final int maxTimeseriesNumberPerBatch =
CONFIG.getMaxLoadingTimeseriesNumber();
private final Map<String, Set<MeasurementSchema>>
currentBatchDevice2TimeseriesSchemas =
new HashMap<>();
- private int currentBatchTimeseriesCount = 0;
private final Set<PartialPath> alreadySetDatabases = new HashSet<>();
private SchemaAutoCreatorAndVerifier() {}
public void autoCreateAndVerify(
- TsFileSequenceReader reader, TimeSeriesIterator timeSeriesIterator)
+ TsFileSequenceReader reader,
+ Map<String, List<TimeseriesMetadata>> device2TimeseriesMetadataList)
throws IOException, AuthException {
- while (currentBatchTimeseriesCount < maxTimeseriesNumberPerBatch
- && timeSeriesIterator.hasNext()) {
- final Pair<String, TimeseriesMetadata> pair =
timeSeriesIterator.next();
- final String device = pair.left;
- final TimeseriesMetadata timeseriesMetadata = pair.right;
- final TSDataType dataType = timeseriesMetadata.getTsDataType();
- if (dataType.equals(TSDataType.VECTOR)) {
- tsfileDevice2IsAligned.put(device, true);
-
- // not a timeseries, skip ++currentBatchTimeseriesCount
- } else {
- // check WRITE_DATA permission of timeseries
- long startTime = System.nanoTime();
- try {
- String userName = context.getSession().getUserName();
- if (!AuthorityChecker.SUPER_USER.equals(userName)) {
- TSStatus status;
- try {
- List<PartialPath> paths =
- Collections.singletonList(
- new PartialPath(device,
timeseriesMetadata.getMeasurementId()));
- status =
- AuthorityChecker.getTSStatus(
- AuthorityChecker.checkFullPathListPermission(
- userName, paths,
PrivilegeType.WRITE_DATA.ordinal()),
- paths,
- PrivilegeType.WRITE_DATA);
- } catch (IllegalPathException e) {
- throw new RuntimeException(e);
- }
- if (status.getCode() !=
TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new AuthException(
- TSStatusCode.representOf(status.getCode()),
status.getMessage());
+ for (final Map.Entry<String, List<TimeseriesMetadata>> entry :
+ device2TimeseriesMetadataList.entrySet()) {
+ final String device = entry.getKey();
+
+ for (final TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
+ final TSDataType dataType = timeseriesMetadata.getTsDataType();
+ if (TSDataType.VECTOR.equals(dataType)) {
+ tsfileDevice2IsAligned.put(device, true);
+
+ // not a timeseries, skip
+ } else {
+ // check WRITE_DATA permission of timeseries
+ long startTime = System.nanoTime();
+ try {
+ String userName = context.getSession().getUserName();
+ if (!AuthorityChecker.SUPER_USER.equals(userName)) {
+ TSStatus status;
+ try {
+ List<PartialPath> paths =
+ Collections.singletonList(
+ new PartialPath(device,
timeseriesMetadata.getMeasurementId()));
+ status =
+ AuthorityChecker.getTSStatus(
+ AuthorityChecker.checkFullPathListPermission(
+ userName, paths,
PrivilegeType.WRITE_DATA.ordinal()),
+ paths,
+ PrivilegeType.WRITE_DATA);
+ } catch (IllegalPathException e) {
+ throw new RuntimeException(e);
+ }
+ if (status.getCode() !=
TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new AuthException(
+ TSStatusCode.representOf(status.getCode()),
status.getMessage());
+ }
}
+ } finally {
+ PerformanceOverviewMetrics.getInstance()
+ .recordAuthCost(System.nanoTime() - startTime);
}
- } finally {
-
PerformanceOverviewMetrics.getInstance().recordAuthCost(System.nanoTime() -
startTime);
+ final Pair<CompressionType, TSEncoding> compressionEncodingPair =
+
reader.readTimeseriesCompressionTypeAndEncoding(timeseriesMetadata);
+ currentBatchDevice2TimeseriesSchemas
+ .computeIfAbsent(device, o -> new HashSet<>())
+ .add(
+ new MeasurementSchema(
+ timeseriesMetadata.getMeasurementId(),
+ dataType,
+ compressionEncodingPair.getRight(),
+ compressionEncodingPair.getLeft()));
+
+ tsfileDevice2IsAligned.putIfAbsent(device, false);
}
- final Pair<CompressionType, TSEncoding> compressionEncodingPair =
-
reader.readTimeseriesCompressionTypeAndEncoding(timeseriesMetadata);
- currentBatchDevice2TimeseriesSchemas
- .computeIfAbsent(device, o -> new HashSet<>())
- .add(
- new MeasurementSchema(
- timeseriesMetadata.getMeasurementId(),
- dataType,
- compressionEncodingPair.getRight(),
- compressionEncodingPair.getLeft()));
-
- tsfileDevice2IsAligned.putIfAbsent(device, false);
-
- ++currentBatchTimeseriesCount;
}
}
- if (currentBatchTimeseriesCount == maxTimeseriesNumberPerBatch) {
- flush();
- }
+ flush();
}
/**
@@ -389,7 +328,6 @@ public class LoadTsfileAnalyzer {
doAutoCreateAndVerify();
currentBatchDevice2TimeseriesSchemas.clear();
- currentBatchTimeseriesCount = 0;
}
private void doAutoCreateAndVerify() throws SemanticException {
@@ -662,7 +600,6 @@ public class LoadTsfileAnalyzer {
public void clear() {
tsfileDevice2IsAligned.clear();
currentBatchDevice2TimeseriesSchemas.clear();
- currentBatchTimeseriesCount = 0;
alreadySetDatabases.clear();
}
}
diff --git
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/TsFileSequenceReaderTimeseriesMetadataIteratorException.java
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/TsFileSequenceReaderTimeseriesMetadataIteratorException.java
new file mode 100644
index 00000000000..a7e2dfc9042
--- /dev/null
+++
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/TsFileSequenceReaderTimeseriesMetadataIteratorException.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.exception;
+
+public class TsFileSequenceReaderTimeseriesMetadataIteratorException extends
RuntimeException {
+
+ public TsFileSequenceReaderTimeseriesMetadataIteratorException(String
message) {
+ super(message);
+ }
+}
diff --git
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/MetadataIndexNodeType.java
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/MetadataIndexNodeType.java
index 20fdc46c3b8..b117a0b632a 100644
---
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/MetadataIndexNodeType.java
+++
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/MetadataIndexNodeType.java
@@ -41,7 +41,7 @@ public enum MetadataIndexNodeType {
/** INTERNAL_MEASUREMENT. */
INTERNAL_MEASUREMENT((byte) 2),
- /** INTERNAL_MEASUREMENT. */
+ /** LEAF_MEASUREMENT. */
LEAF_MEASUREMENT((byte) 3);
private final byte type;
diff --git
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
new file mode 100644
index 00000000000..7648e402e97
--- /dev/null
+++
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.read;
+
+import
org.apache.iotdb.tsfile.exception.TsFileSequenceReaderTimeseriesMetadataIteratorException;
+import org.apache.iotdb.tsfile.file.metadata.MetadataIndexEntry;
+import org.apache.iotdb.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.MetadataIndexNodeType;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+public class TsFileSequenceReaderTimeseriesMetadataIterator
+ implements Iterator<Map<String, List<TimeseriesMetadata>>> {
+
+ private static final int MAX_TIMESERIES_METADATA_COUNT = 2000;
+ private final TsFileSequenceReader reader;
+ private final boolean needChunkMetadata;
+ private ByteBuffer currentBuffer = null;
+ private final Deque<MetadataIndexEntryInfo> metadataIndexEntryStack = new
ArrayDeque<>();
+ private String currentDeviceId;
+ private int currentTimeseriesMetadataCount = 0;
+
+ public TsFileSequenceReaderTimeseriesMetadataIterator(
+ TsFileSequenceReader reader, boolean needChunkMetadata) throws
IOException {
+ this.reader = reader;
+
+ if (this.reader.tsFileMetaData == null) {
+ this.reader.readFileMetadata();
+ }
+
+ final MetadataIndexNode metadataIndexNode =
reader.tsFileMetaData.getMetadataIndex();
+ long curEntryEndOffset = metadataIndexNode.getEndOffset();
+ List<MetadataIndexEntry> metadataIndexEntryList =
metadataIndexNode.getChildren();
+ this.needChunkMetadata = needChunkMetadata;
+
+ for (int i = metadataIndexEntryList.size() - 1; i >= 0; i--) {
+ metadataIndexEntryStack.push(
+ new MetadataIndexEntryInfo(
+ metadataIndexEntryList.get(i), metadataIndexNode.getNodeType(),
curEntryEndOffset));
+ curEntryEndOffset = metadataIndexEntryList.get(i).getOffset();
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !metadataIndexEntryStack.isEmpty()
+ || (currentBuffer != null && currentBuffer.hasRemaining());
+ }
+
+ @Override
+ public Map<String, List<TimeseriesMetadata>> next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+
+ final Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new
HashMap<>();
+
+ while (currentTimeseriesMetadataCount < MAX_TIMESERIES_METADATA_COUNT) {
+ // 1. Check Buffer
+ // currentTimeseriesMetadataCount has reached the limit in the previous
+ // loop and maybe there is still some data that remains in the buffer.
+ if (currentBuffer != null && currentBuffer.hasRemaining()) {
+ timeseriesMetadataMap
+ .computeIfAbsent(currentDeviceId, k -> new ArrayList<>())
+ .addAll(deserializeTimeseriesMetadata());
+ }
+
+ if (currentTimeseriesMetadataCount >= MAX_TIMESERIES_METADATA_COUNT
+ || metadataIndexEntryStack.isEmpty()) {
+ break;
+ }
+
+ // 2. Deserialize MetadataIndexEntry
+ final MetadataIndexEntryInfo indexEntryInfo =
metadataIndexEntryStack.pop();
+
+ try {
+ deserializeMetadataIndexEntry(indexEntryInfo, timeseriesMetadataMap);
+ } catch (IOException e) {
+ throw new TsFileSequenceReaderTimeseriesMetadataIteratorException(
+ String.format(
+ "TsFileSequenceReaderTimeseriesMetadataIterator:
deserializeMetadataIndexEntry failed, "
+ + "MetadataIndexEntryInfo: %s, "
+ + e.getMessage(),
+ indexEntryInfo));
+ }
+ }
+
+ // 3. Reset currentTimeseriesMetadataCount
+ if (currentTimeseriesMetadataCount >= MAX_TIMESERIES_METADATA_COUNT) {
+ currentTimeseriesMetadataCount = 0;
+ }
+
+ return timeseriesMetadataMap;
+ }
+
+ private void deserializeMetadataIndexEntry(
+ MetadataIndexEntryInfo metadataIndexEntryInfo,
+ Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap)
+ throws IOException {
+ if (metadataIndexEntryInfo
+ .getMetadataIndexNodeType()
+ .equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
+ deserializeLeafMeasurement(
+ metadataIndexEntryInfo.getMetadataIndexEntry(),
+ metadataIndexEntryInfo.getEndOffset(),
+ timeseriesMetadataMap);
+
+ } else {
+ deserializeInternalNode(
+ metadataIndexEntryInfo.getMetadataIndexEntry(),
+ metadataIndexEntryInfo.getEndOffset(),
+ metadataIndexEntryInfo
+ .getMetadataIndexNodeType()
+ .equals(MetadataIndexNodeType.LEAF_DEVICE));
+ }
+ }
+
+ private void deserializeLeafMeasurement(
+ MetadataIndexEntry metadataIndexEntry,
+ long endOffset,
+ Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap)
+ throws IOException {
+ if (currentBuffer != null && currentBuffer.hasRemaining()) {
+ throw new TsFileSequenceReaderTimeseriesMetadataIteratorException(
+ "currentBuffer still has some data left before
deserializeLeafMeasurement");
+ }
+
+ currentBuffer = reader.readData(metadataIndexEntry.getOffset(), endOffset);
+
+ timeseriesMetadataMap
+ .computeIfAbsent(currentDeviceId, k -> new ArrayList<>())
+ .addAll(deserializeTimeseriesMetadata());
+ }
+
+ private List<TimeseriesMetadata> deserializeTimeseriesMetadata() {
+ final List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
+ while (currentBuffer.hasRemaining()
+ && currentTimeseriesMetadataCount < MAX_TIMESERIES_METADATA_COUNT) {
+ timeseriesMetadataList.add(
+ TimeseriesMetadata.deserializeFrom(currentBuffer,
needChunkMetadata));
+ currentTimeseriesMetadataCount++;
+ }
+ return timeseriesMetadataList;
+ }
+
+ private void deserializeInternalNode(
+ MetadataIndexEntry metadataIndexEntry, long endOffset, boolean
isLeafDevice)
+ throws IOException {
+ if (isLeafDevice) {
+ currentDeviceId = metadataIndexEntry.getName();
+ }
+
+ final MetadataIndexNode metadataIndexNode =
+ MetadataIndexNode.deserializeFrom(
+ reader.readData(metadataIndexEntry.getOffset(), endOffset));
+ MetadataIndexNodeType metadataIndexNodeType =
metadataIndexNode.getNodeType();
+ List<MetadataIndexEntry> children = metadataIndexNode.getChildren();
+ long curEntryEndOffset = metadataIndexNode.getEndOffset();
+
+ for (int i = children.size() - 1; i >= 0; i--) {
+ metadataIndexEntryStack.push(
+ new MetadataIndexEntryInfo(children.get(i), metadataIndexNodeType,
curEntryEndOffset));
+ curEntryEndOffset = children.get(i).getOffset();
+ }
+ }
+
+ private static class MetadataIndexEntryInfo {
+ private final MetadataIndexEntry metadataIndexEntry;
+ private final MetadataIndexNodeType metadataIndexNodeType;
+ private final long endOffset;
+
+ public MetadataIndexEntryInfo(
+ MetadataIndexEntry metadataIndexEntry,
+ MetadataIndexNodeType metadataIndexNodeType,
+ long endOffset) {
+ this.metadataIndexEntry = metadataIndexEntry;
+ this.metadataIndexNodeType = metadataIndexNodeType;
+ this.endOffset = endOffset;
+ }
+
+ public MetadataIndexEntry getMetadataIndexEntry() {
+ return metadataIndexEntry;
+ }
+
+ public MetadataIndexNodeType getMetadataIndexNodeType() {
+ return metadataIndexNodeType;
+ }
+
+ public long getEndOffset() {
+ return endOffset;
+ }
+ }
+}
diff --git
a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIteratorTest.java
b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIteratorTest.java
new file mode 100644
index 00000000000..54ddaabf03c
--- /dev/null
+++
b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIteratorTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.read;
+
+import org.apache.iotdb.tsfile.utils.FileGenerator;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+public class TsFileSequenceReaderTimeseriesMetadataIteratorTest {
+
+ private static final String FILE_PATH = FileGenerator.outputDataFile;
+ private TsFileReader tsFile;
+
+ @Before
+ public void before() throws IOException {
+ // create 2020 timeseries, 101 measurements per device.
+ FileGenerator.generateFile(100, 20, 101);
+ TsFileSequenceReader fileReader = new TsFileSequenceReader(FILE_PATH);
+ tsFile = new TsFileReader(fileReader);
+ }
+
+ @After
+ public void after() throws IOException {
+ tsFile.close();
+ FileGenerator.after();
+ }
+
+ @Test
+ public void testReadTsFileTimeseriesMetadata() throws IOException {
+ TsFileSequenceReader fileReader = new TsFileSequenceReader(FILE_PATH);
+ TsFileSequenceReaderTimeseriesMetadataIterator iterator =
+ new TsFileSequenceReaderTimeseriesMetadataIterator(fileReader, false);
+
+ Assert.assertTrue(iterator.hasNext());
+ Assert.assertEquals(2000,
iterator.next().values().stream().mapToLong(List::size).sum());
+ Assert.assertTrue(iterator.hasNext());
+ Assert.assertEquals(20,
iterator.next().values().stream().mapToLong(List::size).sum());
+ }
+}
