This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch tsFile_v4
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/tsFile_v4 by this push:
new 1004e15a Enhance compatibility
1004e15a is described below
commit 1004e15a456a8b9ae3e0828bbae75e5a28b2562f
Author: jt2594838 <[email protected]>
AuthorDate: Fri Apr 12 17:54:17 2024 +0800
Enhance compatibility
---
.../apache/tsfile/common/conf/TSFileConfig.java | 2 +-
.../tsfile/compatibility/CompatibilityUtils.java | 47 +---
.../tsfile/compatibility/DeserializeContext.java | 41 +++
.../apache/tsfile/compatibility/Deserializer.java | 26 ++
.../exception/read/FileVersionTooOldException.java | 32 +++
.../tsfile/file/header/ChunkGroupHeader.java | 6 +-
.../file/metadata/DeviceMetadataIndexEntry.java | 20 +-
.../org/apache/tsfile/file/metadata/IDeviceID.java | 3 +
.../file/metadata/MetadataIndexConstructor.java | 2 -
.../tsfile/file/metadata/MetadataIndexNode.java | 6 +-
.../tsfile/file/metadata/StringArrayDeviceID.java | 4 +-
.../apache/tsfile/file/metadata/TableSchema.java | 12 +-
.../tsfile/file/metadata/TsFileMetadata.java | 9 +-
.../apache/tsfile/read/TsFileSequenceReader.java | 274 +++++++++++++--------
...leSequenceReaderTimeseriesMetadataIterator.java | 5 +-
.../read/controller/MetadataQuerierByFileImpl.java | 7 +-
.../reader/block/SingleDeviceTsBlockReader.java | 23 +-
.../reader/series/AbstractFileSeriesReader.java | 2 +-
.../java/org/apache/tsfile/write/TsFileWriter.java | 22 +-
.../org/apache/tsfile/write/record/Tablet.java | 36 +--
.../org/apache/tsfile/write/schema/Schema.java | 11 +-
.../write/writer/tsmiterator/TSMIterator.java | 10 +-
.../tsfile/file/metadata/TsFileMetadataTest.java | 4 +-
.../org/apache/tsfile/read/GetAllDevicesTest.java | 3 +-
...easurementChunkMetadataListMapIteratorTest.java | 3 +-
.../org/apache/tsfile/read/TsFileReaderTest.java | 1 -
.../tsfile/read/TsFileSequenceReaderTest.java | 4 +-
.../tsfile/write/MetadataIndexConstructorTest.java | 10 +-
.../apache/tsfile/write/TableViewWriteTest.java | 47 ++--
.../apache/tsfile/write/TsFileWriteApiTest.java | 5 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 1 -
31 files changed, 428 insertions(+), 250 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
b/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
index 23132894..c4c720dc 100644
--- a/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
@@ -64,7 +64,7 @@ public class TSFileConfig implements Serializable {
public static final String VERSION_NUMBER_V2 = "000002";
public static final String VERSION_NUMBER_V1 = "000001";
/** version number is changed to use 1 byte to represent since version 3. */
- public static final byte VERSION_NUMBER = 0x03;
+ public static final byte VERSION_NUMBER = 0x04;
/** Bloom filter constrain. */
public static final double MIN_BLOOM_FILTER_ERROR_RATE = 0.01;
diff --git
a/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
b/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
index 0933c1ec..2a79cc91 100644
---
a/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
+++
b/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
@@ -19,9 +19,6 @@
package org.apache.tsfile.compatibility;
-import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
-import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry.Deserializer;
-import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.MetadataIndexNode;
import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.file.metadata.TsFileMetadata;
@@ -29,8 +26,6 @@ import org.apache.tsfile.utils.BloomFilter;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
-import java.io.IOException;
-import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Collections;
@@ -40,11 +35,22 @@ public class CompatibilityUtils {
// util class
}
- public static TsFileMetadata deserializeTsFileMetadataFromV3(ByteBuffer
buffer) {
+ public static DeserializeContext v3DeserializeContext = new
DeserializeContext();
+
+ static {
+ v3DeserializeContext.tsFileMetadataDeserializer =
+ CompatibilityUtils::deserializeTsFileMetadataFromV3;
+ v3DeserializeContext.deviceIDDeserializer =
+ ((buffer, context) -> PlainDeviceID.deserialize(buffer));
+ }
+
+ public static TsFileMetadata deserializeTsFileMetadataFromV3(
+ ByteBuffer buffer, DeserializeContext context) {
TsFileMetadata fileMetaData = new TsFileMetadata();
// metadataIndex
- MetadataIndexNode metadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer, true);
+ MetadataIndexNode metadataIndexNode =
+ context.deviceMetadataIndexNodeDeserializer.deserialize(buffer,
context);
fileMetaData.setTableMetadataIndexNodeMap(Collections.singletonMap("",
metadataIndexNode));
// metaOffset
@@ -62,31 +68,4 @@ public class CompatibilityUtils {
return fileMetaData;
}
-
- public static DeviceMetadataIndexEntry.Deserializer
v3DeviceMetadataIndexEntryDeserializer =
- new Deserializer() {
- @Override
- public DeviceMetadataIndexEntry deserializeFrom(ByteBuffer buffer) {
- return deserializeFromV3(buffer);
- }
-
- @Override
- public DeviceMetadataIndexEntry deserializeFrom(InputStream
inputStream)
- throws IOException {
- return deserializeFromV3(inputStream);
- }
- };
-
- public static DeviceMetadataIndexEntry deserializeFromV3(ByteBuffer buffer) {
- IDeviceID device = PlainDeviceID.DESERIALIZER.deserializeFrom(buffer);
- long offset = ReadWriteIOUtils.readLong(buffer);
- return new DeviceMetadataIndexEntry(device, offset);
- }
-
- public static DeviceMetadataIndexEntry deserializeFromV3(InputStream
inputStream)
- throws IOException {
- IDeviceID device = PlainDeviceID.DESERIALIZER.deserializeFrom(inputStream);
- long offset = ReadWriteIOUtils.readLong(inputStream);
- return new DeviceMetadataIndexEntry(device, offset);
- }
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeContext.java
b/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeContext.java
new file mode 100644
index 00000000..0c245cce
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeContext.java
@@ -0,0 +1,41 @@
+package org.apache.tsfile.compatibility;
+
+import org.apache.tsfile.file.IMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.MeasurementMetadataIndexEntry;
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.nio.ByteBuffer;
+
+public class DeserializeContext {
+ public Deserializer<TsFileMetadata> tsFileMetadataDeserializer =
TsFileMetadata::deserializeFrom;
+
+ public Deserializer<MetadataIndexNode> deviceMetadataIndexNodeDeserializer =
+ (buffer, context) -> MetadataIndexNode.deserializeFrom(buffer, true,
context);
+ public Deserializer<MetadataIndexNode>
measurementMetadataIndexNodeDeserializer =
+ (buffer, context) -> MetadataIndexNode.deserializeFrom(buffer, false,
context);
+ public Deserializer<IMetadataIndexEntry>
deviceMetadataIndexEntryDeserializer =
+ DeviceMetadataIndexEntry::deserializeFrom;
+ public Deserializer<IMetadataIndexEntry>
measurementMetadataIndexEntryDeserializer =
+ ((buffer, context) ->
MeasurementMetadataIndexEntry.deserializeFrom(buffer));
+
+ public Deserializer<TableSchema> tableSchemaDeserializer =
TableSchema::deserialize;
+ public Deserializer<MeasurementSchema> measurementSchemaDeserializer =
+ ((buffer, context) -> MeasurementSchema.deserializeFrom(buffer));
+
+ public Deserializer<IDeviceID> deviceIDDeserializer =
+ ((buffer, context) -> StringArrayDeviceID.deserialize(buffer));
+
+ public MetadataIndexNode deserilizeMetadataIndexNode(ByteBuffer buffer,
boolean isDeviceLevel) {
+ if (isDeviceLevel) {
+ return deviceMetadataIndexNodeDeserializer.deserialize(buffer, this);
+ } else {
+ return measurementMetadataIndexNodeDeserializer.deserialize(buffer,
this);
+ }
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/compatibility/Deserializer.java
b/tsfile/src/main/java/org/apache/tsfile/compatibility/Deserializer.java
new file mode 100644
index 00000000..c95eba6e
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/tsfile/compatibility/Deserializer.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.compatibility;
+
+import java.nio.ByteBuffer;
+
+public interface Deserializer<T> {
+ T deserialize(ByteBuffer buffer, DeserializeContext context);
+}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/exception/read/FileVersionTooOldException.java
b/tsfile/src/main/java/org/apache/tsfile/exception/read/FileVersionTooOldException.java
new file mode 100644
index 00000000..2176f7ff
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/tsfile/exception/read/FileVersionTooOldException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.exception.read;
+
+import java.io.IOException;
+
+public class FileVersionTooOldException extends IOException {
+
+ public FileVersionTooOldException(byte currentVersion, byte minimumVersion) {
+ super(
+ String.format(
+ "The current version %d is too old, please at least upgrade to %d",
+ currentVersion, minimumVersion));
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
b/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
index fd9588ea..b9adf3bf 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
@@ -74,7 +74,8 @@ public class ChunkGroupHeader {
}
// TODO: add an interface in IDeviceID
- final IDeviceID deviceID =
IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
+ final IDeviceID deviceID =
+
IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
return new ChunkGroupHeader(deviceID);
}
@@ -96,7 +97,8 @@ public class ChunkGroupHeader {
if (skipped != offsetVar) {
throw new IOException("Skipped " + skipped + " of " + offsetVar);
}
- final IDeviceID deviceID =
IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
+ final IDeviceID deviceID =
+
IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
return new ChunkGroupHeader(deviceID);
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
index 59296db5..f2097664 100644
---
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
+++
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/DeviceMetadataIndexEntry.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.compatibility.DeserializeContext;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -29,20 +30,6 @@ import java.nio.ByteBuffer;
public class DeviceMetadataIndexEntry implements IMetadataIndexEntry {
- public static final Deserializer DESERIALIZER =
- new Deserializer() {
- @Override
- public DeviceMetadataIndexEntry deserializeFrom(ByteBuffer buffer) {
- return DeviceMetadataIndexEntry.deserializeFrom(buffer);
- }
-
- @Override
- public DeviceMetadataIndexEntry deserializeFrom(InputStream
inputStream)
- throws IOException {
- return DeviceMetadataIndexEntry.deserializeFrom(inputStream);
- }
- };
-
private IDeviceID deviceID;
private long offset;
@@ -87,8 +74,9 @@ public class DeviceMetadataIndexEntry implements
IMetadataIndexEntry {
return true;
}
- public static DeviceMetadataIndexEntry deserializeFrom(ByteBuffer buffer) {
- IDeviceID device =
IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(buffer);
+ public static DeviceMetadataIndexEntry deserializeFrom(
+ ByteBuffer buffer, DeserializeContext context) {
+ IDeviceID device = context.deviceIDDeserializer.deserialize(buffer,
context);
long offset = ReadWriteIOUtils.readLong(buffer);
return new DeviceMetadataIndexEntry(device, offset);
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
index 256e3fd2..e7e1b2eb 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
@@ -78,12 +78,15 @@ public interface IDeviceID extends Comparable<IDeviceID>,
Accountable {
interface Deserializer {
IDeviceID deserializeFrom(ByteBuffer byteBuffer);
+
IDeviceID deserializeFrom(InputStream inputStream) throws IOException;
+
Deserializer DEFAULT_DESERIALIZER = StringArrayDeviceID.getDESERIALIZER();
}
interface Factory {
IDeviceID create(String deviceIdString);
+
Factory DEFAULT_FACTORY = StringArrayDeviceID.getFACTORY();
}
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
index 77b07897..b1afd679 100644
---
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
+++
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
@@ -19,7 +19,6 @@
package org.apache.tsfile.file.metadata;
-import java.util.TreeSet;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.file.IMetadataIndexEntry;
@@ -28,7 +27,6 @@ import org.apache.tsfile.write.writer.TsFileOutput;
import java.io.IOException;
import java.util.ArrayDeque;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
index 2034ad4f..95f9ca5a 100644
---
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
+++
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java
@@ -21,6 +21,7 @@ package org.apache.tsfile.file.metadata;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
+import org.apache.tsfile.compatibility.DeserializeContext;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType;
import org.apache.tsfile.utils.Pair;
@@ -98,12 +99,13 @@ public class MetadataIndexNode {
return byteLen;
}
- public static MetadataIndexNode deserializeFrom(ByteBuffer buffer, boolean
isDeviceLevel) {
+ public static MetadataIndexNode deserializeFrom(
+ ByteBuffer buffer, boolean isDeviceLevel, DeserializeContext context) {
List<IMetadataIndexEntry> children = new ArrayList<>();
int size = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
for (int i = 0; i < size; i++) {
if (isDeviceLevel) {
- children.add(DeviceMetadataIndexEntry.deserializeFrom(buffer));
+
children.add(context.deviceMetadataIndexEntryDeserializer.deserialize(buffer,
context));
} else {
children.add(MeasurementMetadataIndexEntry.deserializeFrom(buffer));
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
index 1270962b..7f13cb18 100644
---
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
+++
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java
@@ -19,7 +19,6 @@
package org.apache.tsfile.file.metadata;
-import java.util.Arrays;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.exception.TsFileRuntimeException;
@@ -32,6 +31,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Objects;
public class StringArrayDeviceID implements IDeviceID {
@@ -57,8 +57,6 @@ public class StringArrayDeviceID implements IDeviceID {
}
};
-
-
private static final long INSTANCE_SIZE =
RamUsageEstimator.shallowSizeOfInstance(StringArrayDeviceID.class);
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
index c5625ecc..838eefd0 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.compatibility.DeserializeContext;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.record.Tablet.ColumnType;
import org.apache.tsfile.write.schema.MeasurementSchema;
@@ -123,19 +124,24 @@ public class TableSchema {
return cnt;
}
- public static TableSchema deserialize(String tableName, ByteBuffer buffer) {
+ public static TableSchema deserialize(ByteBuffer buffer, DeserializeContext
context) {
final int tableNum = buffer.getInt();
List<MeasurementSchema> measurementSchemas = new ArrayList<>(tableNum);
List<ColumnType> columnTypes = new ArrayList<>();
for (int i = 0; i < tableNum; i++) {
- MeasurementSchema measurementSchema =
MeasurementSchema.deserializeFrom(buffer);
+ MeasurementSchema measurementSchema =
+ context.measurementSchemaDeserializer.deserialize(buffer, context);
measurementSchemas.add(measurementSchema);
columnTypes.add(ColumnType.values()[buffer.getInt()]);
}
- return new TableSchema(tableName, measurementSchemas, columnTypes);
+ return new TableSchema(null, measurementSchemas, columnTypes);
}
public String getTableName() {
return tableName;
}
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
index eb311766..c8ea5f23 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.compatibility.DeserializeContext;
import org.apache.tsfile.utils.BloomFilter;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -49,7 +50,7 @@ public class TsFileMetadata {
* @param buffer -buffer use to deserialize
* @return -a instance of TsFileMetaData
*/
- public static TsFileMetadata deserializeFrom(ByteBuffer buffer) {
+ public static TsFileMetadata deserializeFrom(ByteBuffer buffer,
DeserializeContext context) {
TsFileMetadata fileMetaData = new TsFileMetadata();
// metadataIndex
@@ -57,7 +58,8 @@ public class TsFileMetadata {
Map<String, MetadataIndexNode> tableIndexNodeMap = new HashMap<>();
for (int i = 0; i < tableIndexNodeNum; i++) {
String tableName = ReadWriteIOUtils.readString(buffer);
- MetadataIndexNode metadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer, true);
+ MetadataIndexNode metadataIndexNode =
+ context.deviceMetadataIndexNodeDeserializer.deserialize(buffer,
context);
tableIndexNodeMap.put(tableName, metadataIndexNode);
}
fileMetaData.setTableMetadataIndexNodeMap(tableIndexNodeMap);
@@ -67,7 +69,8 @@ public class TsFileMetadata {
Map<String, TableSchema> tableSchemaMap = new HashMap<>();
for (int i = 0; i < tableSchemaNum; i++) {
String tableName = ReadWriteIOUtils.readString(buffer);
- TableSchema tableSchema = TableSchema.deserialize(tableName, buffer);
+ TableSchema tableSchema =
context.tableSchemaDeserializer.deserialize(buffer, context);
+ tableSchema.setTableName(tableName);
tableSchemaMap.put(tableName, tableSchema);
}
fileMetaData.setTableSchemaMap(tableSchemaMap);
diff --git
a/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index 68f65290..4ca86fd9 100644
--- a/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -22,12 +22,15 @@ package org.apache.tsfile.read;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.compatibility.CompatibilityUtils;
+import org.apache.tsfile.compatibility.DeserializeContext;
import org.apache.tsfile.compress.IUnCompressor;
import org.apache.tsfile.encoding.decoder.Decoder;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.StopReadTsFileByInterruptException;
import org.apache.tsfile.exception.TsFileRuntimeException;
import org.apache.tsfile.exception.TsFileStatisticsMistakesException;
+import org.apache.tsfile.exception.read.FileVersionTooOldException;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkGroupHeader;
@@ -78,7 +81,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
@@ -120,6 +122,10 @@ public class TsFileSequenceReader implements AutoCloseable
{
private long minPlanIndex = Long.MAX_VALUE;
private long maxPlanIndex = Long.MIN_VALUE;
+ private byte fileVersion;
+
+ private DeserializeContext deserializeContext = new DeserializeContext();
+
/**
* Create a file reader of the given file. The reader will read the tail of
the file to get the
* file metadata size.Then the reader will skip the first
@@ -197,9 +203,9 @@ public class TsFileSequenceReader implements AutoCloseable {
* construct function for TsFileSequenceReader.
*
* @param input the input of a tsfile. The current position should be a
marker and then a chunk
- * Header, rather than the magic number
+ * Header, rather than the magic number
* @param fileMetadataPos the position of the file metadata in the
TsFileInput from the beginning
- * of the input to the current position
+ * of the input to the current position
* @param fileMetadataSize the byte size of the file metadata in the input
*/
public TsFileSequenceReader(TsFileInput input, long fileMetadataPos, int
fileMetadataSize) {
@@ -208,7 +214,32 @@ public class TsFileSequenceReader implements AutoCloseable
{
this.fileMetadataSize = fileMetadataSize;
}
+ private void loadFileVersion() throws IOException {
+
tsFileInput.position(TSFileConfig.MAGIC_STRING.getBytes(TSFileConfig.STRING_CHARSET).length);
+ final ByteBuffer buffer = ByteBuffer.allocate(1);
+ tsFileInput.read(buffer);
+ buffer.flip();
+ fileVersion = buffer.get();
+
+ checkFileVersion();
+ configDeserializer();
+ }
+
+ private void configDeserializer() throws IOException {
+ if (fileVersion == 0x03) {
+ deserializeContext = CompatibilityUtils.v3DeserializeContext;
+ }
+ }
+
+ private void checkFileVersion() throws FileVersionTooOldException {
+ if (TSFileConfig.VERSION_NUMBER - fileVersion > 1) {
+ throw new FileVersionTooOldException(fileVersion, (byte)
(TSFileConfig.VERSION_NUMBER - 1));
+ }
+ }
+
public void loadMetadataSize() throws IOException {
+ loadFileVersion();
+
ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES);
if (readTailMagic().equals(TSFileConfig.MAGIC_STRING)) {
tsFileInput.read(
@@ -233,9 +264,7 @@ public class TsFileSequenceReader implements AutoCloseable {
return fileMetadataSize;
}
- /**
- * Return the tsfile meta data size of this tsfile.
- */
+ /** Return the tsfile meta data size of this tsfile. */
public long getFileMetadataSize() throws IOException {
return tsFileInput.size() - getFileMetadataPos();
}
@@ -251,9 +280,7 @@ public class TsFileSequenceReader implements AutoCloseable {
return tsFileInput.size() - tsFileMetaData.getMetaOffset();
}
- /**
- * this function does not modify the position of the file reader.
- */
+ /** this function does not modify the position of the file reader. */
public String readTailMagic() throws IOException {
long totalSize = tsFileInput.size();
ByteBuffer magicStringBytes =
ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.getBytes().length);
@@ -262,9 +289,7 @@ public class TsFileSequenceReader implements AutoCloseable {
return new String(magicStringBytes.array());
}
- /**
- * whether the file is a complete TsFile: only if the head magic and tail
magic string exists.
- */
+ /** whether the file is a complete TsFile: only if the head magic and tail
magic string exists. */
public boolean isComplete() throws IOException {
long size = tsFileInput.size();
// TSFileConfig.MAGIC_STRING.getBytes().length * 2 for two magic string
@@ -278,9 +303,7 @@ public class TsFileSequenceReader implements AutoCloseable {
}
}
- /**
- * this function does not modify the position of the file reader.
- */
+ /** this function does not modify the position of the file reader. */
public String readHeadMagic() throws IOException {
ByteBuffer magicStringBytes =
ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.getBytes().length);
tsFileInput.read(magicStringBytes, 0);
@@ -288,9 +311,7 @@ public class TsFileSequenceReader implements AutoCloseable {
return new String(magicStringBytes.array());
}
- /**
- * this function reads version number and checks compatibility of TsFile.
- */
+ /** this function reads version number and checks compatibility of TsFile. */
public byte readVersionNumber() throws IOException {
ByteBuffer versionNumberByte = ByteBuffer.allocate(Byte.BYTES);
tsFileInput.read(versionNumberByte,
TSFileConfig.MAGIC_STRING.getBytes().length);
@@ -309,7 +330,8 @@ public class TsFileSequenceReader implements AutoCloseable {
synchronized (this) {
if (tsFileMetaData == null) {
tsFileMetaData =
- TsFileMetadata.deserializeFrom(readData(fileMetadataPos,
fileMetadataSize));
+ deserializeContext.tsFileMetadataDeserializer.deserialize(
+ readData(fileMetadataPos, fileMetadataSize),
deserializeContext);
}
}
}
@@ -400,7 +422,9 @@ public class TsFileSequenceReader implements AutoCloseable {
MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
if
(!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT))
{
try {
- metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
+ metadataIndexNode =
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -462,7 +486,9 @@ public class TsFileSequenceReader implements AutoCloseable {
TimeseriesMetadata firstTimeseriesMetadata;
try {
// next layer MeasurementNode of the specific DeviceNode
- metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
+ metadataIndexNode =
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -566,7 +592,9 @@ public class TsFileSequenceReader implements AutoCloseable {
MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode;
if
(!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT))
{
try {
- metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
+ metadataIndexNode =
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -605,7 +633,9 @@ public class TsFileSequenceReader implements AutoCloseable {
buffer = readData(metadataIndexPair.left.getOffset(),
metadataIndexPair.right);
// next layer MeasurementNode of the specific DeviceNode
try {
- measurementMetadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer, false);
+ measurementMetadataIndexNode =
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -627,20 +657,17 @@ public class TsFileSequenceReader implements
AutoCloseable {
* @throws IOException when read fails
*/
public boolean readITimeseriesMetadata(
- List<TimeseriesMetadata> timeseriesMetadataList,
- MetadataIndexNode node,
- String measurement) throws IOException {
+ List<TimeseriesMetadata> timeseriesMetadataList, MetadataIndexNode node,
String measurement)
+ throws IOException {
Pair<IMetadataIndexEntry, Long> measurementMetadataIndexPair =
- getMetadataAndEndOffsetOfMeasurementNode(
- node, measurement, false);
+ getMetadataAndEndOffsetOfMeasurementNode(node, measurement, false);
if (measurementMetadataIndexPair == null) {
return false;
}
// the content of TimeseriesNode of the specific MeasurementLeafNode
ByteBuffer buffer =
- readData(
- measurementMetadataIndexPair.left.getOffset(),
measurementMetadataIndexPair.right);
+ readData(measurementMetadataIndexPair.left.getOffset(),
measurementMetadataIndexPair.right);
while (buffer.hasRemaining()) {
try {
timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer,
true));
@@ -661,13 +688,16 @@ public class TsFileSequenceReader implements
AutoCloseable {
* @param root search start node, if not provided, use the root node of the
table of the device
* @param mergeAlignedSeries see @return
* @return when the device is not aligned, or mergeAlignedSeries is false,
each result correspond
- * to one series in the provided measurements (if exists); otherwise, all
columns in the aligned
- * device will be merged into one AlignedTimeSeriesMetadata.
+ * to one series in the provided measurements (if exists); otherwise,
all columns in the
+ * aligned device will be merged into one AlignedTimeSeriesMetadata.
* @throws IOException if read fails
*/
public List<ITimeSeriesMetadata> readITimeseriesMetadata(
- IDeviceID device, Set<String> measurements, MetadataIndexNode root,
- boolean mergeAlignedSeries) throws IOException {
+ IDeviceID device,
+ Set<String> measurements,
+ MetadataIndexNode root,
+ boolean mergeAlignedSeries)
+ throws IOException {
// find the index node associated with the device
final MetadataIndexNode measurementMetadataIndexNode =
getDeviceRootNode(device, root);
if (measurementMetadataIndexNode == null) {
@@ -675,8 +705,7 @@ public class TsFileSequenceReader implements AutoCloseable {
}
// Get the time column metadata if the device is aligned
- TimeseriesMetadata timeColumnMetadata =
- getTimeColumnMetadata(measurementMetadataIndexNode);
+ TimeseriesMetadata timeColumnMetadata =
getTimeColumnMetadata(measurementMetadataIndexNode);
List<TimeseriesMetadata> valueTimeseriesMetadataList =
timeColumnMetadata != null ? new ArrayList<>() : null;
@@ -687,32 +716,41 @@ public class TsFileSequenceReader implements
AutoCloseable {
int measurementFoundCnt = 0;
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- for (int i = 0; i < measurementList.size() && measurementFoundCnt <
measurementList.size();
+ for (int i = 0;
+ i < measurementList.size() && measurementFoundCnt <
measurementList.size();
i++) {
final String measurementName = measurementList.get(i);
timeseriesMetadataList.clear();
// read the leaf node that may contain the i-th measurement into a list
if (measurementFound[i]
- || !readITimeseriesMetadata(timeseriesMetadataList,
measurementMetadataIndexNode,
- measurementName)) {
+ || !readITimeseriesMetadata(
+ timeseriesMetadataList, measurementMetadataIndexNode,
measurementName)) {
continue;
}
// in the list, search for the all measurements that are not found
- measurementFoundCnt += searchInTimeseriesList(measurementList, i,
measurementFound,
- timeseriesMetadataList,
- resultTimeseriesMetadataList, timeColumnMetadata,
valueTimeseriesMetadataList,
- mergeAlignedSeries);
+ measurementFoundCnt +=
+ searchInTimeseriesList(
+ measurementList,
+ i,
+ measurementFound,
+ timeseriesMetadataList,
+ resultTimeseriesMetadataList,
+ timeColumnMetadata,
+ valueTimeseriesMetadataList,
+ mergeAlignedSeries);
}
if (valueTimeseriesMetadataList != null &&
!valueTimeseriesMetadataList.isEmpty()) {
resultTimeseriesMetadataList.add(
- new AlignedTimeSeriesMetadata(
- timeColumnMetadata, valueTimeseriesMetadataList));
+ new AlignedTimeSeriesMetadata(timeColumnMetadata,
valueTimeseriesMetadataList));
}
return resultTimeseriesMetadataList;
}
- private int searchInTimeseriesList(List<String> measurementList, int
startIndex,
- boolean[] measurementFound, List<TimeseriesMetadata>
timeseriesMetadataList,
+ private int searchInTimeseriesList(
+ List<String> measurementList,
+ int startIndex,
+ boolean[] measurementFound,
+ List<TimeseriesMetadata> timeseriesMetadataList,
List<ITimeSeriesMetadata> resultTimeseriesMetadataList,
TimeseriesMetadata timeColumnMetadata,
List<TimeseriesMetadata> valueTimeseriesMetadataList,
@@ -720,10 +758,11 @@ public class TsFileSequenceReader implements
AutoCloseable {
int numOfFoundMeasurements = 0;
for (int j = startIndex; j < measurementList.size(); j++) {
int searchResult;
- if (measurementFound[j] ||
- (searchResult =
binarySearchInTimeseriesMetadataList(timeseriesMetadataList,
- measurementList.get(j))) < 0
- ) {
+ if (measurementFound[j]
+ || (searchResult =
+ binarySearchInTimeseriesMetadataList(
+ timeseriesMetadataList, measurementList.get(j)))
+ < 0) {
continue;
}
@@ -745,7 +784,6 @@ public class TsFileSequenceReader implements AutoCloseable {
return numOfFoundMeasurements;
}
-
protected int binarySearchInTimeseriesMetadataList(
List<TimeseriesMetadata> timeseriesMetadataList, String key) {
int low = 0;
@@ -797,7 +835,9 @@ public class TsFileSequenceReader implements AutoCloseable {
endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
}
ByteBuffer buffer =
readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset);
- MetadataIndexNode node = MetadataIndexNode.deserializeFrom(buffer, true);
+ MetadataIndexNode node =
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
deviceList.addAll(getAllDevices(node));
}
return deviceList;
@@ -805,8 +845,8 @@ public class TsFileSequenceReader implements AutoCloseable {
/**
* @return an iterator of "device, isAligned" list, in which names of
devices are ordered in
- * dictionary order, and isAligned represents whether the device is aligned.
Only read devices on
- * one device leaf node each time to save memory.
+ * dictionary order, and isAligned represents whether the device is
aligned. Only read devices
+ * on one device leaf node each time to save memory.
*/
public TsFileDeviceIterator getAllDevicesIteratorWithIsAligned() throws
IOException {
readFileMetadata();
@@ -837,7 +877,9 @@ public class TsFileSequenceReader implements AutoCloseable {
throws IOException {
try {
ByteBuffer nextBuffer = readData(startOffset, endOffset);
- MetadataIndexNode deviceLeafNode =
MetadataIndexNode.deserializeFrom(nextBuffer, true);
+ MetadataIndexNode deviceLeafNode =
+ deserializeContext.deviceMetadataIndexNodeDeserializer.deserialize(
+ nextBuffer, deserializeContext);
getDevicesOfLeafNode(deviceLeafNode, measurementNodeOffsetQueue);
} catch (StopReadTsFileByInterruptException e) {
throw e;
@@ -907,7 +949,9 @@ public class TsFileSequenceReader implements AutoCloseable {
}
ByteBuffer nextBuffer = readData(startOffset, endOffset);
getAllDeviceLeafNodeOffset(
- MetadataIndexNode.deserializeFrom(nextBuffer, true),
leafDeviceNodeOffsets);
+ deserializeContext.deviceMetadataIndexNodeDeserializer.deserialize(
+ nextBuffer, deserializeContext),
+ leafDeviceNodeOffsets);
}
} catch (StopReadTsFileByInterruptException e) {
throw e;
@@ -961,7 +1005,7 @@ public class TsFileSequenceReader implements AutoCloseable
{
/**
* @return an iterator of timeseries list, in which names of timeseries are
ordered in dictionary
- * order
+ * order
* @throws IOException io error
*/
public Iterator<List<Path>> getPathsIterator() throws IOException {
@@ -1025,7 +1069,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
boolean currentChildLevelIsDevice =
MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
MetadataIndexNode metadataIndexNode =
- MetadataIndexNode.deserializeFrom(buffer, currentChildLevelIsDevice);
+ deserializeContext.deserilizeMetadataIndexNode(buffer,
currentChildLevelIsDevice);
+
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
long startOffset = metadataIndexNode.getChildren().get(i).getOffset();
@@ -1064,8 +1109,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
return "".equals(((MeasurementMetadataIndexEntry) entry).getName());
}
- TimeseriesMetadata getTimeColumnMetadata(MetadataIndexNode measurementNode)
- throws IOException {
+ TimeseriesMetadata getTimeColumnMetadata(MetadataIndexNode measurementNode)
throws IOException {
// Not aligned timeseries
if (!isAlignedDevice(measurementNode)) {
return null;
@@ -1090,7 +1134,9 @@ public class TsFileSequenceReader implements
AutoCloseable {
readData(
measurementNode.getChildren().get(0).getOffset(),
measurementNode.getChildren().get(1).getOffset());
- MetadataIndexNode metadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer, false);
+ MetadataIndexNode metadataIndexNode =
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
return getTimeColumnMetadata(metadataIndexNode);
}
return null;
@@ -1102,16 +1148,16 @@ public class TsFileSequenceReader implements
AutoCloseable {
*
* @param measurementNode first measurement node of the device
* @param excludedMeasurementIds do not deserialize chunk metadatas whose
measurementId is in the
- * set. Notice: It only takes effect when the needChunkMetadata parameter is
true.
+ * set. Notice: It only takes effect when the needChunkMetadata
parameter is true.
* @param needChunkMetadata need to deserialize chunk metadatas or not
* @return measurement -> chunk metadata list -> timeseries metadata
<startOffset, endOffset>
*/
public Map<String, Pair<List<IChunkMetadata>, Pair<Long, Long>>>
- getTimeseriesMetadataOffsetByDevice(
- MetadataIndexNode measurementNode,
- Set<String> excludedMeasurementIds,
- boolean needChunkMetadata)
- throws IOException {
+ getTimeseriesMetadataOffsetByDevice(
+ MetadataIndexNode measurementNode,
+ Set<String> excludedMeasurementIds,
+ boolean needChunkMetadata)
+ throws IOException {
Map<String, Pair<List<IChunkMetadata>, Pair<Long, Long>>>
timeseriesMetadataOffsetMap =
new LinkedHashMap<>();
List<IMetadataIndexEntry> childrenEntryList =
measurementNode.getChildren();
@@ -1140,7 +1186,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
} else {
// internal measurement node
MetadataIndexNode nextLayerMeasurementNode =
- MetadataIndexNode.deserializeFrom(nextBuffer, false);
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ nextBuffer, deserializeContext);
timeseriesMetadataOffsetMap.putAll(
getTimeseriesMetadataOffsetByDevice(
nextLayerMeasurementNode, excludedMeasurementIds,
needChunkMetadata));
@@ -1197,7 +1244,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
} else {
// internal measurement node
MetadataIndexNode nextLayerMeasurementNode =
- MetadataIndexNode.deserializeFrom(nextBuffer, false);
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ nextBuffer, deserializeContext);
getDeviceTimeseriesMetadata(
timeseriesMetadataList,
nextLayerMeasurementNode,
@@ -1240,7 +1288,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
boolean currentChildLevelIsDevice =
MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
MetadataIndexNode metadataIndexNode =
- MetadataIndexNode.deserializeFrom(buffer,
currentChildLevelIsDevice);
+ deserializeContext.deserilizeMetadataIndexNode(buffer,
currentChildLevelIsDevice);
+
int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; i++) {
long endOffset = metadataIndexNode.getEndOffset();
@@ -1431,7 +1480,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
* @param metadataIndex given MetadataIndexNode
* @param deviceID target device
* @param exactSearch whether is in exact search mode, return null when
there is no entry with
- * name; or else return the nearest MetadataIndexEntry before it (for deeper
search)
+ * name; or else return the nearest MetadataIndexEntry before it (for
deeper search)
* @return target MetadataIndexEntry, endOffset pair
*/
protected Pair<IMetadataIndexEntry, Long>
getMetadataAndEndOffsetOfDeviceNode(
@@ -1446,7 +1495,10 @@ public class TsFileSequenceReader implements
AutoCloseable {
metadataIndex.getChildIndexEntry(deviceID, false);
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(),
childIndexEntry.right);
return getMetadataAndEndOffsetOfDeviceNode(
- MetadataIndexNode.deserializeFrom(buffer, true), deviceID,
exactSearch);
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext),
+ deviceID,
+ exactSearch);
} else {
return metadataIndex.getChildIndexEntry(deviceID, exactSearch);
}
@@ -1462,7 +1514,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
* @param metadataIndex given MetadataIndexNode
* @param measurement target measurement
* @param exactSearch whether is in exact search mode, return null when
there is no entry with
- * name; or else return the nearest MetadataIndexEntry before it (for deeper
search)
+ * name; or else return the nearest MetadataIndexEntry before it (for
deeper search)
* @return target MetadataIndexEntry, endOffset pair
*/
protected Pair<IMetadataIndexEntry, Long>
getMetadataAndEndOffsetOfMeasurementNode(
@@ -1477,7 +1529,10 @@ public class TsFileSequenceReader implements
AutoCloseable {
metadataIndex.getChildIndexEntry(measurement, false);
ByteBuffer buffer = readData(childIndexEntry.left.getOffset(),
childIndexEntry.right);
return getMetadataAndEndOffsetOfMeasurementNode(
- MetadataIndexNode.deserializeFrom(buffer, false), measurement,
exactSearch);
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext),
+ measurement,
+ exactSearch);
} else {
return metadataIndex.getChildIndexEntry(measurement, exactSearch);
}
@@ -1529,8 +1584,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
/**
- * read data from current position of the input, and deserialize it to a
CHUNK_HEADER. <br> This
- * method is not threadsafe.
+ * read data from current position of the input, and deserialize it to a
CHUNK_HEADER. <br>
+ * This method is not threadsafe.
*
* @return a CHUNK_HEADER
* @throws IOException io error
@@ -1655,9 +1710,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
return
ChunkHeader.deserializeCompressionTypeAndEncoding(tsFileInput.wrapAsInputStream());
}
- /**
- * Get measurement schema by chunkMetadatas.
- */
+ /** Get measurement schema by chunkMetadatas. */
public MeasurementSchema getMeasurementSchema(List<IChunkMetadata>
chunkMetadataList)
throws IOException {
if (chunkMetadataList.isEmpty()) {
@@ -1717,7 +1770,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
/**
- * read one byte from the input. <br> this method is not thread safe
+ * read one byte from the input. <br>
+ * this method is not thread safe
*/
public byte readMarker() throws IOException {
markerBuffer.clear();
@@ -1746,12 +1800,12 @@ public class TsFileSequenceReader implements
AutoCloseable {
/**
* read data from tsFileInput, from the current position (if position = -1),
or the given
- * position. <br> if position = -1, the tsFileInput's position will be
changed to the current
- * position + real data size that been read. Other wise, the tsFileInput's
position is not
- * changed.
+ * position. <br>
+ * if position = -1, the tsFileInput's position will be changed to the
current position + real
+ * data size that been read. Other wise, the tsFileInput's position is not
changed.
*
* @param position the start position of data in the tsFileInput, or the
current position if
- * position = -1
+ * position = -1
* @param totalSize the size of data that want to read
* @return data that been read.
*/
@@ -1792,7 +1846,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
* position.
*
* @param start the start position of data in the tsFileInput, or the
current position if position
- * = -1
+ * = -1
* @param end the end position of data that want to read
* @return data that been read.
*/
@@ -1807,9 +1861,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
}
- /**
- * notice, the target bytebuffer are not flipped.
- */
+ /** notice, the target bytebuffer are not flipped. */
public int readRaw(long position, int length, ByteBuffer target) throws
IOException {
return ReadWriteIOUtils.readAsPossible(tsFileInput, target, position,
length);
}
@@ -1820,9 +1872,9 @@ public class TsFileSequenceReader implements
AutoCloseable {
* @param newSchema the schema on each time series in the file
* @param chunkGroupMetadataList ChunkGroupMetadata List
* @param fastFinish if true and the file is complete, then newSchema and
chunkGroupMetadataList
- * parameter will be not modified.
+ * parameter will be not modified.
* @return the position of the file that is fine. All data after the
position in the file should
- * be truncated.
+ * be truncated.
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
public long selfCheck(
@@ -1903,7 +1955,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
if (dataSize > 0) {
if (((byte) (chunkHeader.getChunkType() & 0x3F))
== MetaMarker
- .CHUNK_HEADER) { // more than one page, we could use page
statistics to
+ .CHUNK_HEADER) { // more than one page, we could use
page statistics to
// generate chunk statistic
while (dataSize > 0) {
// a new Page
@@ -2106,7 +2158,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
*
* @param filename the path of file
* @param fastFinish if true, the method will only check the format of head
(Magic String TsFile,
- * Version Number) and tail (Magic String TsFile) of TsFile.
+ * Version Number) and tail (Magic String TsFile) of TsFile.
* @return the status of TsFile
*/
public long selfCheckWithInfo(
@@ -2290,8 +2342,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
List<List<IChunkMetadata>> results = new
ArrayList<>(timeseriesMetaData.size());
for (ITimeSeriesMetadata timeseriesMetaDatum : timeseriesMetaData) {
List<IChunkMetadata> chunkMetadataList =
readIChunkMetaDataList(timeseriesMetaDatum);
- chunkMetadataList.sort(
- Comparator.comparingLong(IChunkMetadata::getStartTime));
+
chunkMetadataList.sort(Comparator.comparingLong(IChunkMetadata::getStartTime));
results.add(chunkMetadataList);
}
return results;
@@ -2321,7 +2372,9 @@ public class TsFileSequenceReader implements
AutoCloseable {
TimeseriesMetadata firstTimeseriesMetadata;
try {
// next layer MeasurementNode of the specific DeviceNode
- metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer, false);
+ metadataIndexNode =
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
} catch (Exception e) {
logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file);
throw e;
@@ -2484,7 +2537,18 @@ public class TsFileSequenceReader implements
AutoCloseable {
*/
public MetadataIndexNode readMetadataIndexNode(
long startOffset, long endOffset, boolean isDeviceLevel) throws
IOException {
- return MetadataIndexNode.deserializeFrom(readData(startOffset, endOffset),
isDeviceLevel);
+ MetadataIndexNode metadataIndexNode;
+ final ByteBuffer buffer = readData(startOffset, endOffset);
+ if (isDeviceLevel) {
+ metadataIndexNode =
+ deserializeContext.deviceMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
+ } else {
+ metadataIndexNode =
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
+ }
+ return metadataIndexNode;
}
/**
@@ -2532,10 +2596,10 @@ public class TsFileSequenceReader implements
AutoCloseable {
/**
* @return An iterator of linked hashmaps ( measurement -> chunk metadata
list ). When traversing
- * the linked hashmap, you will get chunk metadata lists according to the
lexicographic order of
- * the measurements. The first measurement of the linked hashmap of each
iteration is always
- * larger than the last measurement of the linked hashmap of the previous
iteration in
- * lexicographic order.
+ * the linked hashmap, you will get chunk metadata lists according to
the lexicographic order
+ * of the measurements. The first measurement of the linked hashmap of
each iteration is
+ * always larger than the last measurement of the linked hashmap of the
previous iteration in
+ * lexicographic order.
*/
public Iterator<Map<String, List<ChunkMetadata>>>
getMeasurementChunkMetadataListMapIterator(
IDeviceID device) throws IOException {
@@ -2606,7 +2670,9 @@ public class TsFileSequenceReader implements
AutoCloseable {
private void collectEachLeafMeasurementNodeOffsetRange(
ByteBuffer buffer, Queue<Pair<Long, Long>> queue) throws IOException {
try {
- final MetadataIndexNode metadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer, false);
+ final MetadataIndexNode metadataIndexNode =
+
deserializeContext.measurementMetadataIndexNodeDeserializer.deserialize(
+ buffer, deserializeContext);
final MetadataIndexNodeType metadataIndexNodeType =
metadataIndexNode.getNodeType();
final int metadataIndexListSize = metadataIndexNode.getChildren().size();
for (int i = 0; i < metadataIndexListSize; ++i) {
@@ -2646,4 +2712,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
public int hashCode() {
return Objects.hash(file);
}
+
+ public DeserializeContext getDeserializeContext() {
+ return deserializeContext;
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
b/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
index ec2ca203..933f3f96 100644
---
a/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
+++
b/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
@@ -19,6 +19,7 @@
package org.apache.tsfile.read;
+import org.apache.tsfile.compatibility.DeserializeContext;
import
org.apache.tsfile.exception.TsFileSequenceReaderTimeseriesMetadataIteratorException;
import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.file.metadata.DeviceMetadataIndexEntry;
@@ -50,6 +51,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
private final Deque<MetadataIndexEntryInfo> metadataIndexEntryStack = new
ArrayDeque<>();
private IDeviceID currentDeviceId;
private int currentTimeseriesMetadataCount = 0;
+ private DeserializeContext deserializeContext;
public TsFileSequenceReaderTimeseriesMetadataIterator(
TsFileSequenceReader reader, boolean needChunkMetadata, int
timeseriesBatchReadNumber)
@@ -57,6 +59,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
this.reader = reader;
this.needChunkMetadata = needChunkMetadata;
this.timeseriesBatchReadNumber = timeseriesBatchReadNumber;
+ this.deserializeContext = new DeserializeContext();
if (this.reader.tsFileMetaData == null) {
this.reader.readFileMetadata();
@@ -224,7 +227,7 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
boolean currentChildLevelIsDevice =
MetadataIndexNodeType.INTERNAL_DEVICE.equals(type);
final MetadataIndexNode metadataIndexNode =
- MetadataIndexNode.deserializeFrom(
+ deserializeContext.deserilizeMetadataIndexNode(
reader.readData(metadataIndexEntry.getOffset(), endOffset),
currentChildLevelIsDevice);
MetadataIndexNodeType metadataIndexNodeType =
metadataIndexNode.getNodeType();
List<IMetadataIndexEntry> children = metadataIndexNode.getChildren();
diff --git
a/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
b/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
index af3039ba..00e3ae1c 100644
---
a/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
+++
b/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
@@ -77,7 +77,9 @@ public class MetadataQuerierByFileImpl implements
IMetadataQuerier {
@Override
public List<IChunkMetadata> getChunkMetaDataList(Path timeseriesPath) throws
IOException {
- return new ArrayList<>(deviceIdChunkMetadataCache.get(new
Pair<>(timeseriesPath.getIDeviceID(), timeseriesPath.getMeasurement())));
+ return new ArrayList<>(
+ deviceIdChunkMetadataCache.get(
+ new Pair<>(timeseriesPath.getIDeviceID(),
timeseriesPath.getMeasurement())));
}
public List<List<IChunkMetadata>> getChunkMetadataLists(
@@ -165,7 +167,8 @@ public class MetadataQuerierByFileImpl implements
IMetadataQuerier {
} else {
measurementId = ((TimeseriesMetadata)
timeseriesMetadata).getMeasurementId();
}
- this.deviceIdChunkMetadataCache.put(new Pair<>(selectedDevice,
measurementId), chunkMetadataList);
+ this.deviceIdChunkMetadataCache.put(
+ new Pair<>(selectedDevice, measurementId), chunkMetadataList);
count += chunkMetadataList.size();
if (count == CACHED_ENTRY_NUMBER) {
enough = true;
diff --git
a/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
b/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
index 8099a6ee..42ab35a9 100644
---
a/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
+++
b/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
@@ -13,8 +13,8 @@ import
org.apache.tsfile.read.query.executor.task.DeviceQueryTask;
import org.apache.tsfile.read.reader.series.AbstractFileSeriesReader;
import org.apache.tsfile.read.reader.series.FileSeriesReader;
import org.apache.tsfile.utils.Binary;
-
import org.apache.tsfile.utils.TsPrimitiveType;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,8 +77,9 @@ public class SingleDeviceTsBlockReader implements
TsBlockReader {
}
}
- private void constructColumnContext(List<IChunkMetadata> chunkMetadataList,
- IChunkLoader chunkLoader, Filter timeFilter) throws IOException {
+ private void constructColumnContext(
+ List<IChunkMetadata> chunkMetadataList, IChunkLoader chunkLoader, Filter
timeFilter)
+ throws IOException {
if (chunkMetadataList.isEmpty()) {
return;
}
@@ -87,15 +88,16 @@ public class SingleDeviceTsBlockReader implements
TsBlockReader {
new FileSeriesReader(chunkLoader, chunkMetadataList, timeFilter);
if (seriesReader.hasNextBatch()) {
if (chunkMetadata instanceof AlignedChunkMetadata) {
- final List<String> currentChunkMeasurementNames =
seriesReader.getCurrentChunkMeasurementNames();
+ final List<String> currentChunkMeasurementNames =
+ seriesReader.getCurrentChunkMeasurementNames();
List<List<Integer>> posInResult = new ArrayList<>();
for (String currentChunkMeasurementName :
currentChunkMeasurementNames) {
posInResult.add(task.getColumnMapping().getColumnPos(currentChunkMeasurementName));
}
- measureColumnContextMap.put("",
- new VectorMeasurementColumnContext(posInResult,
- seriesReader.nextBatch(), seriesReader
- ));
+ measureColumnContextMap.put(
+ "",
+ new VectorMeasurementColumnContext(
+ posInResult, seriesReader.nextBatch(), seriesReader));
} else {
final String measurementUid = chunkMetadata.getMeasurementUid();
measureColumnContextMap.put(
@@ -281,8 +283,8 @@ public class SingleDeviceTsBlockReader implements
TsBlockReader {
protected BatchData currentBatch;
protected final AbstractFileSeriesReader seriesReader;
- protected MeasurementColumnContext(AbstractFileSeriesReader seriesReader,
- BatchData currentBatch) {
+ protected MeasurementColumnContext(
+ AbstractFileSeriesReader seriesReader, BatchData currentBatch) {
this.seriesReader = seriesReader;
this.currentBatch = currentBatch;
}
@@ -373,7 +375,6 @@ public class SingleDeviceTsBlockReader implements
TsBlockReader {
}
}
-
public static class IdColumnContext {
private final List<Integer> posInResult;
diff --git
a/tsfile/src/main/java/org/apache/tsfile/read/reader/series/AbstractFileSeriesReader.java
b/tsfile/src/main/java/org/apache/tsfile/read/reader/series/AbstractFileSeriesReader.java
index 0b6a1493..735b44c9 100644
---
a/tsfile/src/main/java/org/apache/tsfile/read/reader/series/AbstractFileSeriesReader.java
+++
b/tsfile/src/main/java/org/apache/tsfile/read/reader/series/AbstractFileSeriesReader.java
@@ -19,7 +19,6 @@
package org.apache.tsfile.read.reader.series;
-import java.util.ArrayList;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.controller.IChunkLoader;
@@ -28,6 +27,7 @@ import org.apache.tsfile.read.reader.IBatchReader;
import org.apache.tsfile.read.reader.IChunkReader;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
/** Series reader is used to query one series of one tsfile. */
diff --git a/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
b/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
index de8d442c..42b06438 100644
--- a/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java
@@ -166,7 +166,8 @@ public class TsFileWriter implements AutoCloseable {
for (Map.Entry<Path, IMeasurementSchema> entry : schemaMap.entrySet()) {
IMeasurementSchema measurementSchema = entry.getValue();
if (measurementSchema instanceof VectorMeasurementSchema) {
- final IDeviceID deviceID =
IDeviceID.Factory.DEFAULT_FACTORY.create(entry.getKey().getDevice());
+ final IDeviceID deviceID =
+
IDeviceID.Factory.DEFAULT_FACTORY.create(entry.getKey().getDevice());
MeasurementGroup group =
measurementGroupMap.getOrDefault(deviceID, new
MeasurementGroup(true));
List<String> measurementList =
measurementSchema.getSubMeasurementsList();
@@ -182,7 +183,8 @@ public class TsFileWriter implements AutoCloseable {
}
measurementGroupMap.put(deviceID, group);
} else {
- final IDeviceID deviceID =
IDeviceID.Factory.DEFAULT_FACTORY.create(entry.getKey().getDevice());
+ final IDeviceID deviceID =
+
IDeviceID.Factory.DEFAULT_FACTORY.create(entry.getKey().getDevice());
MeasurementGroup group =
measurementGroupMap.getOrDefault(deviceID, new
MeasurementGroup(false));
group
@@ -234,7 +236,8 @@ public class TsFileWriter implements AutoCloseable {
@Deprecated
public void registerTimeseries(Path devicePath, MeasurementSchema
measurementSchema)
throws WriteProcessException {
-
registerTimeseries(IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()),
measurementSchema);
+ registerTimeseries(
+ IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()),
measurementSchema);
}
/** Register nonAligned timeseries by single. */
@@ -268,7 +271,8 @@ public class TsFileWriter implements AutoCloseable {
public void registerTimeseries(Path devicePath, List<MeasurementSchema>
measurementSchemas) {
for (MeasurementSchema schema : measurementSchemas) {
try {
-
registerTimeseries(IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()),
schema);
+ registerTimeseries(
+ IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()),
schema);
} catch (WriteProcessException e) {
LOG.warn(e.getMessage());
}
@@ -519,7 +523,10 @@ public class TsFileWriter implements AutoCloseable {
// make sure the ChunkGroupWriter for this Tablet exist
checkIsTimeseriesExist(tablet, false);
// get corresponding ChunkGroupWriter and write this Tablet
- recordCount +=
groupWriters.get(IDeviceID.Factory.DEFAULT_FACTORY.create(tablet.insertTargetName)).write(tablet);
+ recordCount +=
+ groupWriters
+
.get(IDeviceID.Factory.DEFAULT_FACTORY.create(tablet.insertTargetName))
+ .write(tablet);
return checkMemorySizeAndMayFlushChunks();
}
@@ -527,7 +534,10 @@ public class TsFileWriter implements AutoCloseable {
// make sure the ChunkGroupWriter for this Tablet exist
checkIsTimeseriesExist(tablet, true);
// get corresponding ChunkGroupWriter and write this Tablet
- recordCount +=
groupWriters.get(IDeviceID.Factory.DEFAULT_FACTORY.create(tablet.insertTargetName)).write(tablet);
+ recordCount +=
+ groupWriters
+
.get(IDeviceID.Factory.DEFAULT_FACTORY.create(tablet.insertTargetName))
+ .write(tablet);
return checkMemorySizeAndMayFlushChunks();
}
diff --git a/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
b/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
index 65850cbf..1049752b 100644
--- a/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
+++ b/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
@@ -96,27 +96,33 @@ public class Tablet {
}
public Tablet(String insertTargetName, List<MeasurementSchema> schemas, int
maxRowNumber) {
- this(insertTargetName, schemas, ColumnType.nCopy(ColumnType.MEASUREMENT,
schemas.size()),
+ this(
+ insertTargetName,
+ schemas,
+ ColumnType.nCopy(ColumnType.MEASUREMENT, schemas.size()),
maxRowNumber);
}
- public Tablet(String insertTargetName, List<MeasurementSchema> schemas,
- List<ColumnType> columnTypes) {
+ public Tablet(
+ String insertTargetName, List<MeasurementSchema> schemas,
List<ColumnType> columnTypes) {
this(insertTargetName, schemas, columnTypes, DEFAULT_SIZE);
}
- /**
- * Return a {@link Tablet} with the specified number of rows
(maxBatchSize). Only call this
- * constructor directly for testing purposes. {@link Tablet} should
normally always be default
- * size.
- *
- * @param insertTargetName the name of the device specified to be written
in
- * @param schemas the list of {@link MeasurementSchema}s for creating the
row batch, only
- * measurementId and type take effects
- * @param maxRowNumber the maximum number of rows for this tablet
- */
- public Tablet(String insertTargetName, List<MeasurementSchema> schemas,
- List<ColumnType> columnTypes, int maxRowNumber) {
+ /**
+ * Return a {@link Tablet} with the specified number of rows (maxBatchSize).
Only call this
+ * constructor directly for testing purposes. {@link Tablet} should normally
always be default
+ * size.
+ *
+ * @param insertTargetName the name of the device specified to be written in
+ * @param schemas the list of {@link MeasurementSchema}s for creating the
row batch, only
+ * measurementId and type take effects
+ * @param maxRowNumber the maximum number of rows for this tablet
+ */
+ public Tablet(
+ String insertTargetName,
+ List<MeasurementSchema> schemas,
+ List<ColumnType> columnTypes,
+ int maxRowNumber) {
this.insertTargetName = insertTargetName;
this.schemas = new ArrayList<>(schemas);
setColumnTypes(columnTypes);
diff --git a/tsfile/src/main/java/org/apache/tsfile/write/schema/Schema.java
b/tsfile/src/main/java/org/apache/tsfile/write/schema/Schema.java
index 900b287b..9aeb409d 100644
--- a/tsfile/src/main/java/org/apache/tsfile/write/schema/Schema.java
+++ b/tsfile/src/main/java/org/apache/tsfile/write/schema/Schema.java
@@ -57,8 +57,8 @@ public class Schema implements Serializable {
@Deprecated
public void registerTimeseries(Path devicePath, MeasurementSchema
measurementSchema) {
-
registerTimeseries(IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()),
- measurementSchema);
+ registerTimeseries(
+ IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()),
measurementSchema);
}
// This method can only register nonAligned timeseries.
public void registerTimeseries(IDeviceID deviceID, MeasurementSchema
measurementSchema) {
@@ -70,8 +70,8 @@ public class Schema implements Serializable {
@Deprecated
public void registerMeasurementGroup(Path devicePath, MeasurementGroup
measurementGroup) {
-
this.registeredTimeseries.put(IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()),
- measurementGroup);
+ this.registeredTimeseries.put(
+ IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()),
measurementGroup);
}
public void registerMeasurementGroup(IDeviceID deviceID, MeasurementGroup
measurementGroup) {
@@ -117,7 +117,8 @@ public class Schema implements Serializable {
@Deprecated
public MeasurementGroup getSeriesSchema(Path devicePath) {
- return
registeredTimeseries.get(IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()));
+ return registeredTimeseries.get(
+ IDeviceID.Factory.DEFAULT_FACTORY.create(devicePath.getDevice()));
}
public MeasurementGroup getSeriesSchema(IDeviceID devicePath) {
diff --git
a/tsfile/src/main/java/org/apache/tsfile/write/writer/tsmiterator/TSMIterator.java
b/tsfile/src/main/java/org/apache/tsfile/write/writer/tsmiterator/TSMIterator.java
index 530c8581..620cc2da 100644
---
a/tsfile/src/main/java/org/apache/tsfile/write/writer/tsmiterator/TSMIterator.java
+++
b/tsfile/src/main/java/org/apache/tsfile/write/writer/tsmiterator/TSMIterator.java
@@ -120,10 +120,7 @@ public class TSMIterator {
chunkMetadataMap
.get(chunkGroupMetadata.getDevice())
.computeIfAbsent(
- new Path(
- chunkGroupMetadata.getDevice(),
- chunkMetadata.getMeasurementUid(),
- false),
+ new Path(chunkGroupMetadata.getDevice(),
chunkMetadata.getMeasurementUid(), false),
x -> new ArrayList<>())
.add(chunkMetadata);
}
@@ -133,10 +130,7 @@ public class TSMIterator {
chunkMetadataMap
.computeIfAbsent(currentDevice, x -> new TreeMap<>())
.computeIfAbsent(
- new Path(
- currentDevice,
- chunkMetadata.getMeasurementUid(),
- false),
+ new Path(currentDevice, chunkMetadata.getMeasurementUid(),
false),
x -> new ArrayList<>())
.add(chunkMetadata);
}
diff --git
a/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
b/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
index b10712fe..fed04818 100644
---
a/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
+++
b/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.compatibility.DeserializeContext;
import org.apache.tsfile.constant.TestConstant;
import org.apache.tsfile.file.metadata.utils.TestHelper;
import org.apache.tsfile.file.metadata.utils.Utils;
@@ -60,13 +61,14 @@ public class TsFileMetadataTest {
private TsFileMetadata deSerialized() {
FileInputStream fileInputStream = null;
TsFileMetadata metaData = null;
+ DeserializeContext deserializeContext = new DeserializeContext();
try {
fileInputStream = new FileInputStream(new File(PATH));
FileChannel channel = fileInputStream.getChannel();
ByteBuffer buffer = ByteBuffer.allocate((int) channel.size());
channel.read(buffer);
buffer.rewind();
- metaData = TsFileMetadata.deserializeFrom(buffer);
+ metaData = TsFileMetadata.deserializeFrom(buffer, deserializeContext);
return metaData;
} catch (IOException e) {
e.printStackTrace();
diff --git a/tsfile/src/test/java/org/apache/tsfile/read/GetAllDevicesTest.java
b/tsfile/src/test/java/org/apache/tsfile/read/GetAllDevicesTest.java
index 49c0ddb0..3b7811f3 100644
--- a/tsfile/src/test/java/org/apache/tsfile/read/GetAllDevicesTest.java
+++ b/tsfile/src/test/java/org/apache/tsfile/read/GetAllDevicesTest.java
@@ -78,7 +78,8 @@ public class GetAllDevicesTest {
Assert.assertEquals(deviceNum, devices.size());
for (int i = 0; i < deviceNum; i++) {
Assert.assertEquals(
- IDeviceID.Factory.DEFAULT_FACTORY.create("d" +
FileGenerator.generateIndexString(i, deviceNum)),
+ IDeviceID.Factory.DEFAULT_FACTORY.create(
+ "d" + FileGenerator.generateIndexString(i, deviceNum)),
devices.get(i));
}
diff --git
a/tsfile/src/test/java/org/apache/tsfile/read/MeasurementChunkMetadataListMapIteratorTest.java
b/tsfile/src/test/java/org/apache/tsfile/read/MeasurementChunkMetadataListMapIteratorTest.java
index d4fc4e0e..d0bcd9c8 100644
---
a/tsfile/src/test/java/org/apache/tsfile/read/MeasurementChunkMetadataListMapIteratorTest.java
+++
b/tsfile/src/test/java/org/apache/tsfile/read/MeasurementChunkMetadataListMapIteratorTest.java
@@ -157,7 +157,8 @@ public class MeasurementChunkMetadataListMapIteratorTest {
// test not exist device
Iterator<Map<String, List<ChunkMetadata>>> iterator =
-
fileReader.getMeasurementChunkMetadataListMapIterator(IDeviceID.Factory.DEFAULT_FACTORY.create("dd"));
+ fileReader.getMeasurementChunkMetadataListMapIterator(
+ IDeviceID.Factory.DEFAULT_FACTORY.create("dd"));
Assert.assertFalse(iterator.hasNext());
}
diff --git a/tsfile/src/test/java/org/apache/tsfile/read/TsFileReaderTest.java
b/tsfile/src/test/java/org/apache/tsfile/read/TsFileReaderTest.java
index b6298665..f7cb6e44 100644
--- a/tsfile/src/test/java/org/apache/tsfile/read/TsFileReaderTest.java
+++ b/tsfile/src/test/java/org/apache/tsfile/read/TsFileReaderTest.java
@@ -26,7 +26,6 @@ import
org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.common.Path;
diff --git
a/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java
b/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java
index c9fe2663..b07e76ed 100644
--- a/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java
+++ b/tsfile/src/test/java/org/apache/tsfile/read/TsFileSequenceReaderTest.java
@@ -29,7 +29,6 @@ import org.apache.tsfile.file.header.ChunkHeader;
import org.apache.tsfile.file.header.PageHeader;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.utils.FileGenerator;
@@ -134,7 +133,8 @@ public class TsFileSequenceReaderTest {
}
// test for non-exist device "d3"
-
Assert.assertTrue(reader.readChunkMetadataInDevice(IDeviceID.Factory.DEFAULT_FACTORY.create("d3")).isEmpty());
+ Assert.assertTrue(
+
reader.readChunkMetadataInDevice(IDeviceID.Factory.DEFAULT_FACTORY.create("d3")).isEmpty());
reader.close();
}
diff --git
a/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java
b/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java
index 95b206e8..29a7bedd 100644
---
a/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java
+++
b/tsfile/src/test/java/org/apache/tsfile/write/MetadataIndexConstructorTest.java
@@ -139,7 +139,8 @@ public class MetadataIndexConstructorTest {
int[][] vectorMeasurement = new int[deviceNum][];
String[][] singleMeasurement = new String[deviceNum][];
for (int i = 0; i < deviceNum; i++) {
- devices[i] = IDeviceID.Factory.DEFAULT_FACTORY.create("d" +
generateIndexString(i, deviceNum));
+ devices[i] =
+ IDeviceID.Factory.DEFAULT_FACTORY.create("d" +
generateIndexString(i, deviceNum));
vectorMeasurement[i] = new int[0];
singleMeasurement[i] = new String[measurementNum];
for (int j = 0; j < measurementNum; j++) {
@@ -158,7 +159,8 @@ public class MetadataIndexConstructorTest {
int[][] vectorMeasurement = new int[deviceNum][];
String[][] singleMeasurement = new String[deviceNum][];
for (int i = 0; i < deviceNum; i++) {
- devices[i] = IDeviceID.Factory.DEFAULT_FACTORY.create("d" +
generateIndexString(i, deviceNum));
+ devices[i] =
+ IDeviceID.Factory.DEFAULT_FACTORY.create("d" +
generateIndexString(i, deviceNum));
vectorMeasurement[i] = new int[0];
singleMeasurement[i] = new String[measurementNum];
for (int j = 0; j < measurementNum; j++) {
@@ -183,7 +185,9 @@ public class MetadataIndexConstructorTest {
*/
@Test
public void compositeIndexTest() {
- IDeviceID[] devices = {IDeviceID.Factory.DEFAULT_FACTORY.create("d0"),
IDeviceID.Factory.DEFAULT_FACTORY.create("d1")};
+ IDeviceID[] devices = {
+ IDeviceID.Factory.DEFAULT_FACTORY.create("d0"),
IDeviceID.Factory.DEFAULT_FACTORY.create("d1")
+ };
int[][] vectorMeasurement = {{}, {4}};
String[][] singleMeasurement = {
{"s0", "s1", "s2", "s3", "s4", "z0", "z1", "z2", "z3"},
diff --git
a/tsfile/src/test/java/org/apache/tsfile/write/TableViewWriteTest.java
b/tsfile/src/test/java/org/apache/tsfile/write/TableViewWriteTest.java
index d187e8a2..3db8cd9a 100644
--- a/tsfile/src/test/java/org/apache/tsfile/write/TableViewWriteTest.java
+++ b/tsfile/src/test/java/org/apache/tsfile/write/TableViewWriteTest.java
@@ -143,8 +143,8 @@ public class TableViewWriteTest {
for (int i = 0; i < tableNum; i++) {
int cnt;
- try (TsBlockReader reader =
tableQueryExecutor.query(tableSchemas.get(i).getTableName(),
- columns, null, null, null)) {
+ try (TsBlockReader reader =
+ tableQueryExecutor.query(tableSchemas.get(i).getTableName(),
columns, null, null, null)) {
assertTrue(reader.hasNext());
cnt = 0;
while (reader.hasNext()) {
@@ -166,9 +166,9 @@ public class TableViewWriteTest {
final IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.a.b.c.d1");
List<MeasurementSchema> treeSchemas = new ArrayList<>();
for (int i = 0; i < measurementSchemaNum; i++) {
- final MeasurementSchema measurementSchema = new MeasurementSchema("s" +
i, TSDataType.INT64,
- TSEncoding.PLAIN,
- CompressionType.UNCOMPRESSED);
+ final MeasurementSchema measurementSchema =
+ new MeasurementSchema(
+ "s" + i, TSDataType.INT64, TSEncoding.PLAIN,
CompressionType.UNCOMPRESSED);
treeSchemas.add(measurementSchema);
writer.registerTimeseries(deviceID, measurementSchema);
}
@@ -188,8 +188,8 @@ public class TableViewWriteTest {
// table-view read table-view
int cnt;
- try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(
- testFile.getAbsolutePath())) {
+ try (TsFileSequenceReader sequenceReader =
+ new TsFileSequenceReader(testFile.getAbsolutePath())) {
TableQueryExecutor tableQueryExecutor =
new TableQueryExecutor(
new MetadataQuerierByFileImpl(sequenceReader),
@@ -212,11 +212,12 @@ public class TableViewWriteTest {
}
// tree-view read tree-view
- try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(
- testFile.getAbsolutePath())) {
- QueryExecutor queryExecutor = new TsFileExecutor(
- new MetadataQuerierByFileImpl(sequenceReader),
- new CachedChunkLoaderImpl(sequenceReader));
+ try (TsFileSequenceReader sequenceReader =
+ new TsFileSequenceReader(testFile.getAbsolutePath())) {
+ QueryExecutor queryExecutor =
+ new TsFileExecutor(
+ new MetadataQuerierByFileImpl(sequenceReader),
+ new CachedChunkLoaderImpl(sequenceReader));
List<Path> selectedSeries = new ArrayList<>();
for (int i = 0; i < measurementSchemaNum; i++) {
@@ -233,8 +234,8 @@ public class TableViewWriteTest {
}
// table-view read tree-view
- try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(
- testFile.getAbsolutePath())) {
+ try (TsFileSequenceReader sequenceReader =
+ new TsFileSequenceReader(testFile.getAbsolutePath())) {
TableQueryExecutor tableQueryExecutor =
new TableQueryExecutor(
new MetadataQuerierByFileImpl(sequenceReader),
@@ -257,11 +258,12 @@ public class TableViewWriteTest {
}
// tree-view read table-view
- try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(
- testFile.getAbsolutePath())) {
- QueryExecutor queryExecutor = new TsFileExecutor(
- new MetadataQuerierByFileImpl(sequenceReader),
- new CachedChunkLoaderImpl(sequenceReader));
+ try (TsFileSequenceReader sequenceReader =
+ new TsFileSequenceReader(testFile.getAbsolutePath())) {
+ QueryExecutor queryExecutor =
+ new TsFileExecutor(
+ new MetadataQuerierByFileImpl(sequenceReader),
+ new CachedChunkLoaderImpl(sequenceReader));
List<Path> selectedSeries = new ArrayList<>();
for (int i = 0; i < 100; i++) {
@@ -284,8 +286,11 @@ public class TableViewWriteTest {
}
private Tablet genTablet(TableSchema tableSchema, int offset, int num) {
- Tablet tablet = new Tablet(tableSchema.getTableName(),
tableSchema.getColumnSchemas(),
- tableSchema.getColumnTypes());
+ Tablet tablet =
+ new Tablet(
+ tableSchema.getTableName(),
+ tableSchema.getColumnSchemas(),
+ tableSchema.getColumnTypes());
for (int i = 0; i < num; i++) {
tablet.addTimestamp(i, offset + i);
for (MeasurementSchema columnSchema : tableSchema.getColumnSchemas()) {
diff --git
a/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
b/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
index 2bbcf95b..ffb3b01b 100644
--- a/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
+++ b/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
@@ -27,7 +27,6 @@ import org.apache.tsfile.file.header.ChunkHeader;
import org.apache.tsfile.file.header.PageHeader;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.read.TsFileReader;
@@ -633,7 +632,9 @@ public class TsFileWriteApiTest {
TsFileIOWriter tsFileIOWriter = new TsFileIOWriter(file)) {
tsFileIOWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create(deviceId));
for (List<ChunkMetadata> chunkMetadatas :
-
reader.readChunkMetadataInDevice(IDeviceID.Factory.DEFAULT_FACTORY.create(deviceId)).values())
{
+ reader
+
.readChunkMetadataInDevice(IDeviceID.Factory.DEFAULT_FACTORY.create(deviceId))
+ .values()) {
for (ChunkMetadata chunkMetadata : chunkMetadatas) {
Chunk chunk = reader.readMemChunk(chunkMetadata);
ByteBuffer chunkDataBuffer = chunk.getData();
diff --git
a/tsfile/src/test/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriterTest.java
b/tsfile/src/test/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriterTest.java
index 6b12b38e..d6d0f942 100644
---
a/tsfile/src/test/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriterTest.java
+++
b/tsfile/src/test/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriterTest.java
@@ -25,7 +25,6 @@ import
org.apache.tsfile.exception.NotCompatibleTsFileException;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.file.metadata.statistics.FloatStatistics;
