This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch iotdb in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 470eef5de6a82778fb91f44b9d3b9aa6b4766939 Author: Zikun Ma <[email protected]> AuthorDate: Mon Sep 9 17:48:32 2024 +0800 Load: deserialize PlainDeviceID for TsFile V3 (#231) --- .../tsfile/compatibility/CompatibilityUtils.java | 7 +++++++ .../tsfile/compatibility/DeserializeConfig.java | 1 + .../tsfile/file/header/ChunkGroupHeader.java | 23 ++++++++++++++-------- .../org/apache/tsfile/file/metadata/IDeviceID.java | 1 + .../apache/tsfile/file/metadata/PlainDeviceID.java | 17 ++++++++++++++++ .../apache/tsfile/read/TsFileSequenceReader.java | 6 ++++-- ...leSequenceReaderTimeseriesMetadataIterator.java | 2 +- 7 files changed, 46 insertions(+), 11 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java index de01351f..ec0102c5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java @@ -38,6 +38,8 @@ public class CompatibilityUtils { public static DeserializeConfig v3DeserializeConfig = new DeserializeConfig(); static { + v3DeserializeConfig.versionNumber = + org.apache.tsfile.common.conf.TSFileConfig.VERSION_NUMBER_V3; v3DeserializeConfig.tsFileMetadataBufferDeserializer = CompatibilityUtils::deserializeTsFileMetadataFromV3; v3DeserializeConfig.deviceIDBufferDeserializer = @@ -45,6 +47,11 @@ public class CompatibilityUtils { final PlainDeviceID deviceID = PlainDeviceID.deserialize(buffer); return deviceID.convertToStringArrayDeviceId(); }); + v3DeserializeConfig.deviceIDStreamDeserializer = + ((stream, context) -> { + final PlainDeviceID deviceID = PlainDeviceID.deserialize(stream); + return deviceID.convertToStringArrayDeviceId(); + }); } public static TsFileMetadata deserializeTsFileMetadataFromV3( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java index b1c4f18f..b9228bd0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java @@ -34,6 +34,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; public class DeserializeConfig { + public byte versionNumber = org.apache.tsfile.common.conf.TSFileConfig.VERSION_NUMBER; public BufferDeserializer<TsFileMetadata> tsFileMetadataBufferDeserializer = TsFileMetadata::deserializeFrom; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java b/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java index 7a424f89..b46f0e0c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java @@ -64,8 +64,8 @@ public class ChunkGroupHeader { * @param markerRead - Whether the marker of the CHUNK_GROUP_HEADER is read ahead. * @throws IOException – If an I/O error occurs. */ - public static ChunkGroupHeader deserializeFrom(InputStream inputStream, boolean markerRead) - throws IOException { + public static ChunkGroupHeader deserializeFrom( + InputStream inputStream, boolean markerRead, byte versionNumber) throws IOException { if (!markerRead) { byte marker = (byte) inputStream.read(); if (marker != MARKER) { @@ -74,8 +74,7 @@ public class ChunkGroupHeader { } // TODO: add an interface in IDeviceID - final IDeviceID deviceID = - IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream); + final IDeviceID deviceID = deserializeDeviceID(inputStream, versionNumber); return new ChunkGroupHeader(deviceID); } @@ -85,19 +84,27 @@ public class ChunkGroupHeader { * @param markerRead - Whether the marker of the CHUNK_GROUP_HEADER is read ahead. * @throws IOException - If an I/O error occurs. */ - public static ChunkGroupHeader deserializeFrom(TsFileInput input, long offset, boolean markerRead) - throws IOException { + public static ChunkGroupHeader deserializeFrom( + TsFileInput input, long offset, boolean markerRead, byte versionNumber) throws IOException { long offsetVar = offset; if (!markerRead) { offsetVar++; } input.position(offsetVar); final InputStream inputStream = input.wrapAsInputStream(); - final IDeviceID deviceID = - IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream); + final IDeviceID deviceID = deserializeDeviceID(inputStream, versionNumber); return new ChunkGroupHeader(deviceID); } + private static IDeviceID deserializeDeviceID(InputStream inputStream, byte versionNumber) + throws IOException { + final IDeviceID.Deserializer deserializer = + versionNumber == org.apache.tsfile.common.conf.TSFileConfig.VERSION_NUMBER + ? IDeviceID.Deserializer.DEFAULT_DESERIALIZER + : IDeviceID.Deserializer.DESERIALIZER_V3; + return deserializer.deserializeFrom(inputStream); + } + public IDeviceID getDeviceID() { return deviceID; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java index 87e04cc0..b67053a5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java @@ -151,6 +151,7 @@ public interface IDeviceID extends Comparable<IDeviceID>, Accountable, Serializa IDeviceID deserializeFrom(InputStream inputStream) throws IOException; Deserializer DEFAULT_DESERIALIZER = StringArrayDeviceID.getDESERIALIZER(); + Deserializer DESERIALIZER_V3 = PlainDeviceID.getDESERIALIZER(); } interface Factory { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java index f36255fe..cc93350f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java @@ -48,6 +48,23 @@ public class PlainDeviceID implements IDeviceID { this.deviceID = deviceID; } + private static final Deserializer DESERIALIZER = + new Deserializer() { + @Override + public IDeviceID deserializeFrom(ByteBuffer byteBuffer) { + return deserialize(byteBuffer).convertToStringArrayDeviceId(); + } + + @Override + public IDeviceID deserializeFrom(InputStream inputStream) throws IOException { + return deserialize(inputStream).convertToStringArrayDeviceId(); + } + }; + + public static Deserializer getDESERIALIZER() { + return DESERIALIZER; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 57c2500d..9daff0c1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -1479,7 +1479,8 @@ public class TsFileSequenceReader implements AutoCloseable { * @throws IOException io error */ public ChunkGroupHeader readChunkGroupHeader() throws IOException { - return ChunkGroupHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), true); + return ChunkGroupHeader.deserializeFrom( + tsFileInput.wrapAsInputStream(), true, deserializeConfig.versionNumber); } /** @@ -1492,7 +1493,8 @@ public class TsFileSequenceReader implements AutoCloseable { */ public ChunkGroupHeader readChunkGroupHeader(long position, boolean markerRead) throws IOException { - return ChunkGroupHeader.deserializeFrom(tsFileInput, position, markerRead); + return ChunkGroupHeader.deserializeFrom( + tsFileInput, position, markerRead, deserializeConfig.versionNumber); } public void readPlanIndex() throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java index 9fa1c311..798714a8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java @@ -59,7 +59,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator this.reader = reader; this.needChunkMetadata = needChunkMetadata; this.timeseriesBatchReadNumber = timeseriesBatchReadNumber; - this.deserializeConfig = new DeserializeConfig(); + this.deserializeConfig = this.reader.getDeserializeContext(); if (this.reader.tsFileMetaData == null) { this.reader.readFileMetadata();
