This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new a90fe9ac Load: deserialize PlainDeviceID for TsFile V3 (#231)
a90fe9ac is described below
commit a90fe9ac18db2b422dc3e02f964e0a81c192470e
Author: Zikun Ma <[email protected]>
AuthorDate: Mon Sep 9 17:48:32 2024 +0800
Load: deserialize PlainDeviceID for TsFile V3 (#231)
---
.../tsfile/compatibility/CompatibilityUtils.java | 7 +++++++
.../tsfile/compatibility/DeserializeConfig.java | 1 +
.../tsfile/file/header/ChunkGroupHeader.java | 23 ++++++++++++++--------
.../org/apache/tsfile/file/metadata/IDeviceID.java | 1 +
.../apache/tsfile/file/metadata/PlainDeviceID.java | 17 ++++++++++++++++
.../apache/tsfile/read/TsFileSequenceReader.java | 6 ++++--
...leSequenceReaderTimeseriesMetadataIterator.java | 2 +-
7 files changed, 46 insertions(+), 11 deletions(-)
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
index de01351f..ec0102c5 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
@@ -38,6 +38,8 @@ public class CompatibilityUtils {
public static DeserializeConfig v3DeserializeConfig = new
DeserializeConfig();
static {
+ v3DeserializeConfig.versionNumber =
+ org.apache.tsfile.common.conf.TSFileConfig.VERSION_NUMBER_V3;
v3DeserializeConfig.tsFileMetadataBufferDeserializer =
CompatibilityUtils::deserializeTsFileMetadataFromV3;
v3DeserializeConfig.deviceIDBufferDeserializer =
@@ -45,6 +47,11 @@ public class CompatibilityUtils {
final PlainDeviceID deviceID = PlainDeviceID.deserialize(buffer);
return deviceID.convertToStringArrayDeviceId();
});
+ v3DeserializeConfig.deviceIDStreamDeserializer =
+ ((stream, context) -> {
+ final PlainDeviceID deviceID = PlainDeviceID.deserialize(stream);
+ return deviceID.convertToStringArrayDeviceId();
+ });
}
public static TsFileMetadata deserializeTsFileMetadataFromV3(
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
index b1c4f18f..b9228bd0 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
@@ -34,6 +34,7 @@ import java.io.InputStream;
import java.nio.ByteBuffer;
public class DeserializeConfig {
+ public byte versionNumber =
org.apache.tsfile.common.conf.TSFileConfig.VERSION_NUMBER;
public BufferDeserializer<TsFileMetadata> tsFileMetadataBufferDeserializer =
TsFileMetadata::deserializeFrom;
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
index 7a424f89..b46f0e0c 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
@@ -64,8 +64,8 @@ public class ChunkGroupHeader {
* @param markerRead - Whether the marker of the CHUNK_GROUP_HEADER is read
ahead.
* @throws IOException – If an I/O error occurs.
*/
- public static ChunkGroupHeader deserializeFrom(InputStream inputStream,
boolean markerRead)
- throws IOException {
+ public static ChunkGroupHeader deserializeFrom(
+ InputStream inputStream, boolean markerRead, byte versionNumber) throws
IOException {
if (!markerRead) {
byte marker = (byte) inputStream.read();
if (marker != MARKER) {
@@ -74,8 +74,7 @@ public class ChunkGroupHeader {
}
// TODO: add an interface in IDeviceID
- final IDeviceID deviceID =
-
IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
+ final IDeviceID deviceID = deserializeDeviceID(inputStream, versionNumber);
return new ChunkGroupHeader(deviceID);
}
@@ -85,19 +84,27 @@ public class ChunkGroupHeader {
* @param markerRead - Whether the marker of the CHUNK_GROUP_HEADER is read
ahead.
* @throws IOException - If an I/O error occurs.
*/
- public static ChunkGroupHeader deserializeFrom(TsFileInput input, long
offset, boolean markerRead)
- throws IOException {
+ public static ChunkGroupHeader deserializeFrom(
+ TsFileInput input, long offset, boolean markerRead, byte versionNumber)
throws IOException {
long offsetVar = offset;
if (!markerRead) {
offsetVar++;
}
input.position(offsetVar);
final InputStream inputStream = input.wrapAsInputStream();
- final IDeviceID deviceID =
-
IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
+ final IDeviceID deviceID = deserializeDeviceID(inputStream, versionNumber);
return new ChunkGroupHeader(deviceID);
}
+ private static IDeviceID deserializeDeviceID(InputStream inputStream, byte
versionNumber)
+ throws IOException {
+ final IDeviceID.Deserializer deserializer =
+ versionNumber ==
org.apache.tsfile.common.conf.TSFileConfig.VERSION_NUMBER
+ ? IDeviceID.Deserializer.DEFAULT_DESERIALIZER
+ : IDeviceID.Deserializer.DESERIALIZER_V3;
+ return deserializer.deserializeFrom(inputStream);
+ }
+
public IDeviceID getDeviceID() {
return deviceID;
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
index 87e04cc0..b67053a5 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
@@ -151,6 +151,7 @@ public interface IDeviceID extends Comparable<IDeviceID>,
Accountable, Serializa
IDeviceID deserializeFrom(InputStream inputStream) throws IOException;
Deserializer DEFAULT_DESERIALIZER = StringArrayDeviceID.getDESERIALIZER();
+ Deserializer DESERIALIZER_V3 = PlainDeviceID.getDESERIALIZER();
}
interface Factory {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
index f36255fe..cc93350f 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
@@ -48,6 +48,23 @@ public class PlainDeviceID implements IDeviceID {
this.deviceID = deviceID;
}
+ private static final Deserializer DESERIALIZER =
+ new Deserializer() {
+ @Override
+ public IDeviceID deserializeFrom(ByteBuffer byteBuffer) {
+ return deserialize(byteBuffer).convertToStringArrayDeviceId();
+ }
+
+ @Override
+ public IDeviceID deserializeFrom(InputStream inputStream) throws
IOException {
+ return deserialize(inputStream).convertToStringArrayDeviceId();
+ }
+ };
+
+ public static Deserializer getDESERIALIZER() {
+ return DESERIALIZER;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index 57c2500d..9daff0c1 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -1479,7 +1479,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
* @throws IOException io error
*/
public ChunkGroupHeader readChunkGroupHeader() throws IOException {
- return ChunkGroupHeader.deserializeFrom(tsFileInput.wrapAsInputStream(),
true);
+ return ChunkGroupHeader.deserializeFrom(
+ tsFileInput.wrapAsInputStream(), true,
deserializeConfig.versionNumber);
}
/**
@@ -1492,7 +1493,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
*/
public ChunkGroupHeader readChunkGroupHeader(long position, boolean
markerRead)
throws IOException {
- return ChunkGroupHeader.deserializeFrom(tsFileInput, position, markerRead);
+ return ChunkGroupHeader.deserializeFrom(
+ tsFileInput, position, markerRead, deserializeConfig.versionNumber);
}
public void readPlanIndex() throws IOException {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
index 9fa1c311..798714a8 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
@@ -59,7 +59,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
this.reader = reader;
this.needChunkMetadata = needChunkMetadata;
this.timeseriesBatchReadNumber = timeseriesBatchReadNumber;
- this.deserializeConfig = new DeserializeConfig();
+ this.deserializeConfig = this.reader.getDeserializeContext();
if (this.reader.tsFileMetaData == null) {
this.reader.readFileMetadata();
