This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch tsFile_v4
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/tsFile_v4 by this push:
new fec211e9 add stream deserializer
fec211e9 is described below
commit fec211e99fbcb4b8de57ae7b17b80c3088272fc7
Author: jt2594838 <[email protected]>
AuthorDate: Fri Apr 12 18:26:53 2024 +0800
add stream deserializer
---
.../{Deserializer.java => BufferDeserializer.java} | 4 +-
.../tsfile/compatibility/CompatibilityUtils.java | 15 ++--
.../tsfile/compatibility/DeserializeConfig.java | 83 ++++++++++++++++++++
.../tsfile/compatibility/DeserializeContext.java | 41 ----------
.../tsfile/compatibility/StreamDeserializer.java | 8 ++
.../tsfile/file/header/ChunkGroupHeader.java | 1 -
.../file/metadata/DeviceMetadataIndexEntry.java | 17 ++---
.../tsfile/file/metadata/MetadataIndexNode.java | 19 ++---
.../apache/tsfile/file/metadata/PlainDeviceID.java | 29 ++-----
.../apache/tsfile/file/metadata/TableSchema.java | 6 +-
.../tsfile/file/metadata/TsFileMetadata.java | 8 +-
.../apache/tsfile/read/TsFileSequenceReader.java | 84 ++++++++++-----------
...leSequenceReaderTimeseriesMetadataIterator.java | 8 +-
.../tsfile/compatibility/CompatibilityTest.java | 55 ++++++++++++++
.../tsfile/file/metadata/TsFileMetadataTest.java | 6 +-
tsfile/src/test/resources/v3TsFile | Bin 0 -> 378 bytes
16 files changed, 231 insertions(+), 153 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/tsfile/compatibility/Deserializer.java
b/tsfile/src/main/java/org/apache/tsfile/compatibility/BufferDeserializer.java
similarity index 89%
rename from
tsfile/src/main/java/org/apache/tsfile/compatibility/Deserializer.java
rename to
tsfile/src/main/java/org/apache/tsfile/compatibility/BufferDeserializer.java
index c95eba6e..d82182d6 100644
--- a/tsfile/src/main/java/org/apache/tsfile/compatibility/Deserializer.java
+++
b/tsfile/src/main/java/org/apache/tsfile/compatibility/BufferDeserializer.java
@@ -21,6 +21,6 @@ package org.apache.tsfile.compatibility;
import java.nio.ByteBuffer;
-public interface Deserializer<T> {
- T deserialize(ByteBuffer buffer, DeserializeContext context);
+public interface BufferDeserializer<T> {
+ T deserialize(ByteBuffer buffer, DeserializeConfig context);
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
b/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
index 2a79cc91..de01351f 100644
---
a/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
+++
b/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
@@ -35,22 +35,25 @@ public class CompatibilityUtils {
// util class
}
- public static DeserializeContext v3DeserializeContext = new
DeserializeContext();
+ public static DeserializeConfig v3DeserializeConfig = new
DeserializeConfig();
static {
- v3DeserializeContext.tsFileMetadataDeserializer =
+ v3DeserializeConfig.tsFileMetadataBufferDeserializer =
CompatibilityUtils::deserializeTsFileMetadataFromV3;
- v3DeserializeContext.deviceIDDeserializer =
- ((buffer, context) -> PlainDeviceID.deserialize(buffer));
+ v3DeserializeConfig.deviceIDBufferDeserializer =
+ ((buffer, context) -> {
+ final PlainDeviceID deviceID = PlainDeviceID.deserialize(buffer);
+ return deviceID.convertToStringArrayDeviceId();
+ });
}
public static TsFileMetadata deserializeTsFileMetadataFromV3(
- ByteBuffer buffer, DeserializeContext context) {
+ ByteBuffer buffer, DeserializeConfig context) {
TsFileMetadata fileMetaData = new TsFileMetadata();
// metadataIndex
MetadataIndexNode metadataIndexNode =
- context.deviceMetadataIndexNodeDeserializer.deserialize(buffer,
context);
+ context.deviceMetadataIndexNodeBufferDeserializer.deserialize(buffer,
context);
fileMetaData.setTableMetadataIndexNodeMap(Collections.singletonMap("",
metadataIndexNode));
// metaOffset
diff --git
a/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
b/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
new file mode 100644
index 00000000..ac0b8afa
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
@@ -0,0 +1,83 @@
+package org.apache.tsfile.compatibility;
+
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.tsfile.file.IMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.MeasurementMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.nio.ByteBuffer;
+
+public class DeserializeConfig {
+ public BufferDeserializer<TsFileMetadata> tsFileMetadataBufferDeserializer =
TsFileMetadata::deserializeFrom;
+
+ public BufferDeserializer<MetadataIndexNode>
deviceMetadataIndexNodeBufferDeserializer =
+ (buffer, context) -> MetadataIndexNode.deserializeFrom(buffer, true,
context);
+ public BufferDeserializer<MetadataIndexNode>
measurementMetadataIndexNodeBufferDeserializer =
+ (buffer, context) -> MetadataIndexNode.deserializeFrom(buffer, false,
context);
+ public BufferDeserializer<IMetadataIndexEntry>
deviceMetadataIndexEntryBufferDeserializer =
+ DeviceMetadataIndexEntry::deserializeFrom;
+ public BufferDeserializer<IMetadataIndexEntry>
measurementMetadataIndexEntryBufferDeserializer =
+ ((buffer, context) ->
MeasurementMetadataIndexEntry.deserializeFrom(buffer));
+
+ public BufferDeserializer<TableSchema> tableSchemaBufferDeserializer =
TableSchema::deserialize;
+ public BufferDeserializer<MeasurementSchema>
measurementSchemaBufferDeserializer =
+ ((buffer, context) -> MeasurementSchema.deserializeFrom(buffer));
+
+ public BufferDeserializer<IDeviceID> deviceIDBufferDeserializer =
+ ((buffer, context) -> StringArrayDeviceID.deserialize(buffer));
+
+ // stream deserializers
+ public StreamDeserializer<MetadataIndexNode>
deviceMetadataIndexNodeStreamDeserializer =
+ (stream, context) -> MetadataIndexNode.deserializeFrom(stream, true,
context);
+ public StreamDeserializer<MetadataIndexNode>
measurementMetadataIndexNodeStreamDeserializer =
+ (stream, context) -> MetadataIndexNode.deserializeFrom(stream, false,
context);
+ public StreamDeserializer<IMetadataIndexEntry>
deviceMetadataIndexEntryStreamDeserializer =
+ DeviceMetadataIndexEntry::deserializeFrom;
+ public StreamDeserializer<IMetadataIndexEntry>
measurementMetadataIndexEntryStreamDeserializer =
+ ((stream, context) ->
MeasurementMetadataIndexEntry.deserializeFrom(stream));
+
+ public StreamDeserializer<IDeviceID> deviceIDStreamDeserializer =
+ ((stream, context) -> StringArrayDeviceID.deserialize(stream));
+
+ public MetadataIndexNode deserializeMetadataIndexNode(ByteBuffer buffer,
boolean isDeviceLevel) {
+ if (isDeviceLevel) {
+ return deviceMetadataIndexNodeBufferDeserializer.deserialize(buffer,
this);
+ } else {
+ return
measurementMetadataIndexNodeBufferDeserializer.deserialize(buffer, this);
+ }
+ }
+
+ public IMetadataIndexEntry deserializeMetadataIndexEntry(ByteBuffer buffer,
+ boolean isDeviceLevel) {
+ if (isDeviceLevel) {
+ return deviceMetadataIndexEntryBufferDeserializer.deserialize(buffer,
this);
+ } else {
+ return
measurementMetadataIndexEntryBufferDeserializer.deserialize(buffer, this);
+ }
+ }
+
+ public MetadataIndexNode deserializeMetadataIndexNode(InputStream stream,
boolean isDeviceLevel)
+ throws IOException {
+ if (isDeviceLevel) {
+ return deviceMetadataIndexNodeStreamDeserializer.deserialize(stream,
this);
+ } else {
+ return
measurementMetadataIndexNodeStreamDeserializer.deserialize(stream, this);
+ }
+ }
+
+ public IMetadataIndexEntry deserializeMetadataIndexEntry(InputStream stream,
+ boolean isDeviceLevel) throws IOException {
+ if (isDeviceLevel) {
+ return deviceMetadataIndexEntryStreamDeserializer.deserialize(stream,
this);
+ } else {
+ return
measurementMetadataIndexEntryStreamDeserializer.deserialize(stream, this);
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeContext.java
b/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeContext.java
deleted file mode 100644
index 0c245cce..00000000
---
a/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeContext.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.apache.tsfile.compatibility;
-
-import org.apache.tsfile.file.IMetadataIndexEntry;
-import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.MeasurementMetadataIndexEntry;
-import org.apache.tsfile.file.metadata.MetadataIndexNode;
-import org.apache.tsfile.file.metadata.StringArrayDeviceID;
-import org.apache.tsfile.file.metadata.TableSchema;
-import org.apache.tsfile.file.metadata.TsFileMetadata;
-import org.apache.tsfile.write.schema.MeasurementSchema;
-
-import java.nio.ByteBuffer;
-
-public class DeserializeContext {
- public Deserializer<TsFileMetadata> tsFileMetadataDeserializer =
TsFileMetadata::deserializeFrom;
-
- public Deserializer<MetadataIndexNode> deviceMetadataIndexNodeDeserializer =
- (buffer, context) -> MetadataIndexNode.deserializeFrom(buffer, true,
context);
- public Deserializer<MetadataIndexNode>
measurementMetadataIndexNodeDeserializer =
- (buffer, context) -> MetadataIndexNode.deserializeFrom(buffer, false,
context);
- public Deserializer<IMetadataIndexEntry>
deviceMetadataIndexEntryDeserializer =
- DeviceMetadataIndexEntry::deserializeFrom;
- public Deserializer<IMetadataIndexEntry>
measurementMetadataIndexEntryDeserializer =
- ((buffer, context) ->
MeasurementMetadataIndexEntry.deserializeFrom(buffer));
-
- public Deserializer<TableSchema> tableSchemaDeserializer =
TableSchema::deserialize;
- public Deserializer<MeasurementSchema> measurementSchemaDeserializer =
- ((buffer, context) -> MeasurementSchema.deserializeFrom(buffer));
-
- public Deserializer<IDeviceID> deviceIDDeserializer =
- ((buffer, context) -> StringArrayDeviceID.deserialize(buffer));
-
- public MetadataIndexNode deserilizeMetadataIndexNode(ByteBuffer buffer,
boolean isDeviceLevel) {
- if (isDeviceLevel) {
- return deviceMetadataIndexNodeDeserializer.deserialize(buffer, this);
- } else {
- return measurementMetadataIndexNodeDeserializer.deserialize(buffer,
this);
- }
- }
-}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/compatibility/StreamDeserializer.java
b/tsfile/src/main/java/org/apache/tsfile/compatibility/StreamDeserializer.java
new file mode 100644
index 00000000..b0afd457
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/tsfile/compatibility/StreamDeserializer.java
@@ -0,0 +1,8 @@
+package org.apache.tsfile.compatibility;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public interface StreamDeserializer<T> {
+ T deserialize(InputStream inputStream, DeserializeConfig context) throws
IOException;
+}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
b/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
index b9adf3bf..6bc0493e 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
@@ -91,7 +91,6 @@ public class ChunkGroupHeader {
if (!markerRead) {
offsetVar++;
}
- // TODO: add an interface in IDeviceID
final InputStream inputStream = input.wrapAsInputStream();
final long skipped = inputStream.skip(offsetVar);
if (skipped != offsetVar) {
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
index f2097664..d483bc5d 100644
---
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
+++
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
@@ -19,7 +19,7 @@
package org.apache.tsfile.file.metadata;
-import org.apache.tsfile.compatibility.DeserializeContext;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -75,15 +75,16 @@ public class DeviceMetadataIndexEntry implements
IMetadataIndexEntry {
}
public static DeviceMetadataIndexEntry deserializeFrom(
- ByteBuffer buffer, DeserializeContext context) {
- IDeviceID device = context.deviceIDDeserializer.deserialize(buffer,
context);
+ ByteBuffer buffer, DeserializeConfig context) {
+ IDeviceID device = context.deviceIDBufferDeserializer.deserialize(buffer,
context);
long offset = ReadWriteIOUtils.readLong(buffer);
return new DeviceMetadataIndexEntry(device, offset);
}
- public static DeviceMetadataIndexEntry deserializeFrom(InputStream
inputStream)
+ public static DeviceMetadataIndexEntry deserializeFrom(InputStream
inputStream,
+ DeserializeConfig config)
throws IOException {
- IDeviceID device =
IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
+ IDeviceID device =
config.deviceIDStreamDeserializer.deserialize(inputStream, config);
long offset = ReadWriteIOUtils.readLong(inputStream);
return new DeviceMetadataIndexEntry(device, offset);
}
@@ -92,10 +93,4 @@ public class DeviceMetadataIndexEntry implements
IMetadataIndexEntry {
public String toString() {
return "<" + deviceID + "," + offset + ">";
}
-
- public interface Deserializer {
- DeviceMetadataIndexEntry deserializeFrom(ByteBuffer buffer);
-
- DeviceMetadataIndexEntry deserializeFrom(InputStream inputStream) throws
IOException;
- }
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
index 95f9ca5a..e55cf490 100644
---
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
+++
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
@@ -21,7 +21,7 @@ package org.apache.tsfile.file.metadata;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
-import org.apache.tsfile.compatibility.DeserializeContext;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType;
import org.apache.tsfile.utils.Pair;
@@ -100,15 +100,11 @@ public class MetadataIndexNode {
}
public static MetadataIndexNode deserializeFrom(
- ByteBuffer buffer, boolean isDeviceLevel, DeserializeContext context) {
+ ByteBuffer buffer, boolean isDeviceLevel, DeserializeConfig context) {
List<IMetadataIndexEntry> children = new ArrayList<>();
int size = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
for (int i = 0; i < size; i++) {
- if (isDeviceLevel) {
-
children.add(context.deviceMetadataIndexEntryDeserializer.deserialize(buffer,
context));
- } else {
- children.add(MeasurementMetadataIndexEntry.deserializeFrom(buffer));
- }
+ children.add(context.deserializeMetadataIndexEntry(buffer,
isDeviceLevel));
}
long offset = ReadWriteIOUtils.readLong(buffer);
MetadataIndexNodeType nodeType =
@@ -116,16 +112,13 @@ public class MetadataIndexNode {
return new MetadataIndexNode(children, offset, nodeType);
}
- public static MetadataIndexNode deserializeFrom(InputStream inputStream,
boolean isDeviceLevel)
+ public static MetadataIndexNode deserializeFrom(InputStream inputStream,
boolean isDeviceLevel,
+ DeserializeConfig config)
throws IOException {
List<IMetadataIndexEntry> children = new ArrayList<>();
int size = ReadWriteForEncodingUtils.readUnsignedVarInt(inputStream);
for (int i = 0; i < size; i++) {
- if (isDeviceLevel) {
- children.add(DeviceMetadataIndexEntry.deserializeFrom(inputStream));
- } else {
-
children.add(MeasurementMetadataIndexEntry.deserializeFrom(inputStream));
- }
+ children.add(config.deserializeMetadataIndexEntry(inputStream,
isDeviceLevel));
}
long offset = ReadWriteIOUtils.readLong(inputStream);
MetadataIndexNodeType nodeType =
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
index 0cf77051..61d32dcf 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
@@ -35,27 +35,6 @@ import static
org.apache.tsfile.utils.RamUsageEstimator.sizeOfCharArray;
/** Using device id path as id. */
public class PlainDeviceID implements IDeviceID {
- public static final Deserializer DESERIALIZER =
- new Deserializer() {
- @Override
- public IDeviceID deserializeFrom(ByteBuffer byteBuffer) {
- return deserialize(byteBuffer);
- }
-
- @Override
- public IDeviceID deserializeFrom(InputStream inputStream) throws
IOException {
- return deserialize(inputStream);
- }
- };
-
- public static final Factory FACTORY =
- new Factory() {
- @Override
- public IDeviceID create(String deviceIdString) {
- return new PlainDeviceID(deviceIdString);
- }
- };
-
// TODO: configurable but unchangeable
private static final int DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3;
private static final long INSTANCE_SIZE =
@@ -128,14 +107,18 @@ public class PlainDeviceID implements IDeviceID {
return size;
}
- public static IDeviceID deserialize(ByteBuffer byteBuffer) {
+ public static PlainDeviceID deserialize(ByteBuffer byteBuffer) {
return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(byteBuffer));
}
- public static IDeviceID deserialize(InputStream inputStream) throws
IOException {
+ public static PlainDeviceID deserialize(InputStream inputStream) throws
IOException {
return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(inputStream));
}
+ public StringArrayDeviceID convertToStringArrayDeviceId() {
+ return new StringArrayDeviceID(deviceID);
+ }
+
@Override
public int compareTo(IDeviceID other) {
if (!(other instanceof PlainDeviceID)) {
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
index 838eefd0..7005d779 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
@@ -19,7 +19,7 @@
package org.apache.tsfile.file.metadata;
-import org.apache.tsfile.compatibility.DeserializeContext;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.record.Tablet.ColumnType;
import org.apache.tsfile.write.schema.MeasurementSchema;
@@ -124,13 +124,13 @@ public class TableSchema {
return cnt;
}
- public static TableSchema deserialize(ByteBuffer buffer, DeserializeContext
context) {
+ public static TableSchema deserialize(ByteBuffer buffer, DeserializeConfig
context) {
final int tableNum = buffer.getInt();
List<MeasurementSchema> measurementSchemas = new ArrayList<>(tableNum);
List<ColumnType> columnTypes = new ArrayList<>();
for (int i = 0; i < tableNum; i++) {
MeasurementSchema measurementSchema =
- context.measurementSchemaDeserializer.deserialize(buffer, context);
+ context.measurementSchemaBufferDeserializer.deserialize(buffer,
context);
measurementSchemas.add(measurementSchema);
columnTypes.add(ColumnType.values()[buffer.getInt()]);
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
index c8ea5f23..f0dfcf53 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
@@ -19,7 +19,7 @@
package org.apache.tsfile.file.metadata;
-import org.apache.tsfile.compatibility.DeserializeContext;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.utils.BloomFilter;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -50,7 +50,7 @@ public class TsFileMetadata {
* @param buffer -buffer use to deserialize
* @return -a instance of TsFileMetaData
*/
- public static TsFileMetadata deserializeFrom(ByteBuffer buffer,
DeserializeContext context) {
+ public static TsFileMetadata deserializeFrom(ByteBuffer buffer,
DeserializeConfig context) {
TsFileMetadata fileMetaData = new TsFileMetadata();
// metadataIndex
@@ -59,7 +59,7 @@ public class TsFileMetadata {
for (int i = 0; i < tableIndexNodeNum; i++) {
String tableName = ReadWriteIOUtils.readString(buffer);
MetadataIndexNode metadataIndexNode =
- context.deviceMetadataIndexNodeDeserializer.deserialize(buffer,
context);
+
context.deviceMetadataIndexNodeBufferDeserializer.deserialize(buffer, context);
tableIndexNodeMap.put(tableName, metadataIndexNode);
}
fileMetaData.setTableMetadataIndexNodeMap(tableIndexNodeMap);
@@ -69,7 +69,7 @@ public class TsFileMetadata {
Map<String, TableSchema> tableSchemaMap = new HashMap<>();
for (int i = 0; i < tableSchemaNum; i++) {
String tableName = ReadWriteIOUtils.readString(buffer);
- TableSchema tableSchema =
context.tableSchemaDeserializer.deserialize(buffer, context);
+ TableSchema tableSchema =
context.tableSchemaBufferDeserializer.deserialize(buffer, context);
tableSchema.setTableName(tableName);
tableSchemaMap.put(tableName, tableSchema);
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index 4ca86fd9..eb2de930 100644
--- a/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -23,7 +23,7 @@ import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.compatibility.CompatibilityUtils;
-import org.apache.tsfile.compatibility.DeserializeContext;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.compress.IUnCompressor;
import org.apache.tsfile.encoding.decoder.Decoder;
import org.apache.tsfile.enums.TSDataType;
@@ -124,7 +124,7 @@ public class TsFileSequenceReader implements AutoCloseable {
private byte fileVersion;
- private DeserializeContext deserializeContext = new DeserializeContext();
+ private DeserializeConfig deserializeConfig = new DeserializeConfig();
/**
* Create a file reader of the given file. The reader will read the tail of
the file to get the
@@ -227,7 +227,7 @@ public class TsFileSequenceReader implements AutoCloseable {
private void configDeserializer() throws IOException {
if (fileVersion == 0x03) {
- deserializeContext = CompatibilityUtils.v3DeserializeContext;
+ deserializeConfig = CompatibilityUtils.v3DeserializeConfig;
}
}
@@ -330,8 +330,8 @@ public class TsFileSequenceReader implements AutoCloseable {
synchronized (this) {
if (tsFileMetaData == null) {
tsFileMetaData =
- deserializeContext.tsFileMetadataDeserializer.deserialize(
- readData(fileMetadataPos, fileMetadataSize),
deserializeContext);
+ deserializeConfig.tsFileMetadataBufferDeserializer.deserialize(
+ readData(fileMetadataPos, fileMetadataSize),
deserializeConfig);
}
}
}
@@ -423,8 +423,8 @@ public class TsFileSequenceReader implements AutoCloseable {
if
(!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT))
{
try {
metadataIndexNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -487,8 +487,8 @@ public class TsFileSequenceReader implements AutoCloseable {
try {
// next layer MeasurementNode of the specific DeviceNode
metadataIndexNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -593,8 +593,8 @@ public class TsFileSequenceReader implements AutoCloseable {
if
(!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT))
{
try {
metadataIndexNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -634,8 +634,8 @@ public class TsFileSequenceReader implements AutoCloseable {
// next layer MeasurementNode of the specific DeviceNode
try {
measurementMetadataIndexNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -836,8 +836,8 @@ public class TsFileSequenceReader implements AutoCloseable {
}
ByteBuffer buffer =
readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset);
MetadataIndexNode node =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
deviceList.addAll(getAllDevices(node));
}
return deviceList;
@@ -878,8 +878,8 @@ public class TsFileSequenceReader implements AutoCloseable {
try {
ByteBuffer nextBuffer = readData(startOffset, endOffset);
MetadataIndexNode deviceLeafNode =
- deserializeContext.deviceMetadataIndexNodeDeserializer.deserialize(
- nextBuffer, deserializeContext);
+
deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
+ nextBuffer, deserializeConfig);
getDevicesOfLeafNode(deviceLeafNode, measurementNodeOffsetQueue);
} catch (StopReadTsFileByInterruptException e) {
throw e;
@@ -949,8 +949,8 @@ public class TsFileSequenceReader implements AutoCloseable {
}
ByteBuffer nextBuffer = readData(startOffset, endOffset);
getAllDeviceLeafNodeOffset(
- deserializeContext.deviceMetadataIndexNodeDeserializer.deserialize(
- nextBuffer, deserializeContext),
+
deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
+ nextBuffer, deserializeConfig),
leafDeviceNodeOffsets);
}
} catch (StopReadTsFileByInterruptException e) {
@@ -1069,7 +1069,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
boolean currentChildLevelIsDevice =
MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
MetadataIndexNode metadataIndexNode =
- deserializeContext.deserilizeMetadataIndexNode(buffer,
currentChildLevelIsDevice);
+ deserializeConfig.deserializeMetadataIndexNode(buffer,
currentChildLevelIsDevice);
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
@@ -1135,8 +1135,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
measurementNode.getChildren().get(0).getOffset(),
measurementNode.getChildren().get(1).getOffset());
MetadataIndexNode metadataIndexNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
return getTimeColumnMetadata(metadataIndexNode);
}
return null;
@@ -1186,8 +1186,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
} else {
// internal measurement node
MetadataIndexNode nextLayerMeasurementNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- nextBuffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ nextBuffer, deserializeConfig);
timeseriesMetadataOffsetMap.putAll(
getTimeseriesMetadataOffsetByDevice(
nextLayerMeasurementNode, excludedMeasurementIds,
needChunkMetadata));
@@ -1244,8 +1244,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
} else {
// internal measurement node
MetadataIndexNode nextLayerMeasurementNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- nextBuffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ nextBuffer, deserializeConfig);
getDeviceTimeseriesMetadata(
timeseriesMetadataList,
nextLayerMeasurementNode,
@@ -1288,7 +1288,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
boolean currentChildLevelIsDevice =
MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
MetadataIndexNode metadataIndexNode =
- deserializeContext.deserilizeMetadataIndexNode(buffer,
currentChildLevelIsDevice);
+ deserializeConfig.deserializeMetadataIndexNode(buffer,
currentChildLevelIsDevice);
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
@@ -1355,7 +1355,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
boolean currentChildLevelIsDevice =
MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
MetadataIndexNode metadataIndexNode =
- MetadataIndexNode.deserializeFrom(
+ deserializeConfig.deserializeMetadataIndexNode(
tsFileInput.wrapAsInputStream(), currentChildLevelIsDevice);
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
@@ -1495,8 +1495,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
metadataIndex.getChildIndexEntry(deviceID, false);
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(),
childIndexEntry.right);
return getMetadataAndEndOffsetOfDeviceNode(
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext),
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig),
deviceID,
exactSearch);
} else {
@@ -1529,8 +1529,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
metadataIndex.getChildIndexEntry(measurement, false);
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(),
childIndexEntry.right);
return getMetadataAndEndOffsetOfMeasurementNode(
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext),
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig),
measurement,
exactSearch);
} else {
@@ -2373,8 +2373,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
try {
// next layer MeasurementNode of the specific DeviceNode
metadataIndexNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -2541,12 +2541,12 @@ public class TsFileSequenceReader implements
AutoCloseable {
final ByteBuffer buffer = readData(startOffset, endOffset);
if (isDeviceLevel) {
metadataIndexNode =
- deserializeContext.deviceMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.deviceMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
} else {
metadataIndexNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
}
return metadataIndexNode;
}
@@ -2671,8 +2671,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
ByteBuffer buffer, Queue<Pair<Long, Long>> queue) throws IOException {
try {
final MetadataIndexNode metadataIndexNode =
-
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
- buffer, deserializeContext);
+
deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize(
+ buffer, deserializeConfig);
final MetadataIndexNodeType metadataIndexNodeType =
metadataIndexNode.getNodeType();
final int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; ++i) {
@@ -2713,7 +2713,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
return Objects.hash(file);
}
- public DeserializeContext getDeserializeContext() {
- return deserializeContext;
+ public DeserializeConfig getDeserializeContext() {
+ return deserializeConfig;
}
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
b/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
index 933f3f96..9fa1c311 100644
---
a/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
+++
b/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
@@ -19,7 +19,7 @@
package org.apache.tsfile.read;
-import org.apache.tsfile.compatibility.DeserializeContext;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import
org.apache.tsfile.exception.TsFileSequenceReaderTimeseriesMetadataIteratorException;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
@@ -51,7 +51,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
private final Deque<MetadataIndexEntryInfo> metadataIndexEntryStack = new
ArrayDeque<>();
private IDeviceID currentDeviceId;
private int currentTimeseriesMetadataCount = 0;
- private DeserializeContext deserializeContext;
+ private DeserializeConfig deserializeConfig;
public TsFileSequenceReaderTimeseriesMetadataIterator(
TsFileSequenceReader reader, boolean needChunkMetadata, int
timeseriesBatchReadNumber)
@@ -59,7 +59,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
this.reader = reader;
this.needChunkMetadata = needChunkMetadata;
this.timeseriesBatchReadNumber = timeseriesBatchReadNumber;
- this.deserializeContext = new DeserializeContext();
+ this.deserializeConfig = new DeserializeConfig();
if (this.reader.tsFileMetaData == null) {
this.reader.readFileMetadata();
@@ -227,7 +227,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
boolean currentChildLevelIsDevice =
MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
final MetadataIndexNode metadataIndexNode =
- deserializeContext.deserilizeMetadataIndexNode(
+ deserializeConfig.deserializeMetadataIndexNode(
reader.readData(metadataIndexEntry.getOffset(), endOffset),
currentChildLevelIsDevice);
MetadataIndexNodeType metadataIndexNodeType =
metadataIndexNode.getNodeType();
List<IMetadataIndexEntry> children = metadataIndexNode.getChildren();
diff --git
a/tsfile/src/test/java/org/apache/tsfile/compatibility/CompatibilityTest.java
b/tsfile/src/test/java/org/apache/tsfile/compatibility/CompatibilityTest.java
new file mode 100644
index 00000000..9df39ded
--- /dev/null
+++
b/tsfile/src/test/java/org/apache/tsfile/compatibility/CompatibilityTest.java
@@ -0,0 +1,55 @@
+package org.apache.tsfile.compatibility;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.tsfile.read.TsFileReader;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.read.common.RowRecord;
+import org.apache.tsfile.read.expression.QueryExpression;
+import org.apache.tsfile.read.query.dataset.QueryDataSet;
+import org.junit.Test;
+
+public class CompatibilityTest {
+
+ private String fileName = "target" + File.separator + "test-classes" +
File.separator +
+ "v3TsFile";
+
+ /**
+ * The file is generated by the TsFileWriter in version 3.
+ */
+ @Test
+ public void testReadV3() {
+ readOneRow();
+ }
+
+ private void readOneRow() {
+ readOneRow(5);
+ }
+
+ private void readOneRow(int s2Value) {
+ try {
+ TsFileReader tsFileReader = new TsFileReader(new
TsFileSequenceReader(fileName));
+ QueryDataSet dataSet =
+ tsFileReader.query(
+ QueryExpression.create()
+ .addSelectedPath(new Path("d1", "s1", true))
+ .addSelectedPath(new Path("d1", "s2", true))
+ .addSelectedPath(new Path("d1", "s3", true)));
+ while (dataSet.hasNext()) {
+ RowRecord result = dataSet.next();
+ assertEquals(2, result.getFields().size());
+ assertEquals(10000, result.getTimestamp());
+ assertEquals(5.0f, result.getFields().get(0).getFloatV(), 0.00001);
+ assertEquals(s2Value, result.getFields().get(1).getIntV());
+ }
+ tsFileReader.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
diff --git
a/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
b/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
index fed04818..cfac37e8 100644
---
a/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
+++
b/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.tsfile.file.metadata;
-import org.apache.tsfile.compatibility.DeserializeContext;
+import org.apache.tsfile.compatibility.DeserializeConfig;
import org.apache.tsfile.constant.TestConstant;
import org.apache.tsfile.file.metadata.utils.TestHelper;
import org.apache.tsfile.file.metadata.utils.Utils;
@@ -61,14 +61,14 @@ public class TsFileMetadataTest {
private TsFileMetadata deSerialized() {
FileInputStream fileInputStream = null;
TsFileMetadata metaData = null;
- DeserializeContext deserializeContext = new DeserializeContext();
+ DeserializeConfig deserializeConfig = new DeserializeConfig();
try {
fileInputStream = new FileInputStream(new File(PATH));
FileChannel channel = fileInputStream.getChannel();
ByteBuffer buffer = ByteBuffer.allocate((int) channel.size());
channel.read(buffer);
buffer.rewind();
- metaData = TsFileMetadata.deserializeFrom(buffer, deserializeContext);
+ metaData = TsFileMetadata.deserializeFrom(buffer, deserializeConfig);
return metaData;
} catch (IOException e) {
e.printStackTrace();
diff --git a/tsfile/src/test/resources/v3TsFile
b/tsfile/src/test/resources/v3TsFile
new file mode 100644
index 00000000..5502286b
Binary files /dev/null and b/tsfile/src/test/resources/v3TsFile differ
