This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch tsFile_v4
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/tsFile_v4 by this push:
new 22774e0d Add ser/deser of TableSchema
22774e0d is described below
commit 22774e0d4fd7f4999f28609d3f4cd7da0c24b9dd
Author: jt2594838 <[email protected]>
AuthorDate: Sun Apr 7 15:52:55 2024 +0800
Add ser/deser of TableSchema
---
.../file/metadata/MetadataIndexConstructor.java | 12 +++++
.../apache/tsfile/file/metadata/TableSchema.java | 29 ++++++++++++
.../tsfile/file/metadata/TsFileMetadata.java | 50 +++++++++++++++++----
.../apache/tsfile/utils/CompatibilityUtils.java | 51 ++++++++++++++++++++++
.../org/apache/tsfile/write/schema/Schema.java | 4 ++
.../apache/tsfile/write/writer/TsFileIOWriter.java | 14 +++++-
6 files changed, 149 insertions(+), 11 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
index 3941ec4f..875efe79 100644
---
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
+++
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexConstructor.java
@@ -27,6 +27,7 @@ import org.apache.tsfile.write.writer.TsFileOutput;
import java.io.IOException;
import java.util.ArrayDeque;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -88,6 +89,17 @@ public class MetadataIndexConstructor {
return checkAndBuildLevelIndex(deviceMetadataIndexMap, out);
}
+ public static Map<String, Map<IDeviceID, MetadataIndexNode>>
splitDeviceByTable(
+ Map<IDeviceID, MetadataIndexNode> deviceMetadataIndexMap) {
+ Map<String, Map<IDeviceID, MetadataIndexNode>> result = new HashMap<>();
+ for (Entry<IDeviceID, MetadataIndexNode> entry :
deviceMetadataIndexMap.entrySet()) {
+ IDeviceID deviceID = entry.getKey();
+ String tableName = deviceID.getTableName();
+ result.computeIfAbsent(tableName, tName -> new
HashMap<>()).put(deviceID, entry.getValue());
+ }
+ return result;
+ }
+
public static MetadataIndexNode checkAndBuildLevelIndex(
Map<IDeviceID, MetadataIndexNode> deviceMetadataIndexMap, TsFileOutput
out)
throws IOException {
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
index 0741fd55..b67621d4 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
@@ -19,14 +19,20 @@
package org.apache.tsfile.file.metadata;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class TableSchema {
+ // the tableName is not serialized since the TableSchema is always stored in
a Map, from whose
+ // key the tableName can be known
protected String tableName;
protected List<MeasurementSchema> columnSchemas;
protected boolean updatable = false;
@@ -84,4 +90,27 @@ public class TableSchema {
public List<MeasurementSchema> getColumnSchemas() {
return columnSchemas;
}
+
+ public int serialize(OutputStream out) throws IOException {
+ int cnt = 0;
+ if (columnSchemas != null) {
+ cnt += ReadWriteIOUtils.write(columnSchemas.size(), out);
+ for (MeasurementSchema columnSchema : columnSchemas) {
+ cnt += columnSchema.serializeTo(out);
+ }
+ } else {
+ cnt += ReadWriteIOUtils.write(0, out);
+ }
+ return cnt;
+ }
+
+ public static TableSchema deserialize(String tableName, ByteBuffer buffer) {
+ final int tableNum = buffer.getInt();
+ List<MeasurementSchema> measurementSchemas = new ArrayList<>(tableNum);
+ for (int i = 0; i < tableNum; i++) {
+ MeasurementSchema measurementSchema =
MeasurementSchema.deserializeFrom(buffer);
+ measurementSchemas.add(measurementSchema);
+ }
+ return new TableSchema(tableName, measurementSchemas);
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
index 726a1148..5cec83ef 100644
--- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
+++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
@@ -26,7 +26,9 @@ import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import java.util.HashMap;
import java.util.Map;
+import java.util.Map.Entry;
/** TSFileMetaData collects all metadata info and saves in its data structure.
*/
public class TsFileMetadata {
@@ -35,7 +37,6 @@ public class TsFileMetadata {
private BloomFilter bloomFilter;
// List of <name, offset, childMetadataIndexType>
- private MetadataIndexNode metadataIndex;
private Map<String, MetadataIndexNode> tableMetadataIndexNodeMap;
private Map<String, TableSchema> tableSchemaMap;
@@ -52,7 +53,24 @@ public class TsFileMetadata {
TsFileMetadata fileMetaData = new TsFileMetadata();
// metadataIndex
- fileMetaData.metadataIndex = MetadataIndexNode.deserializeFrom(buffer,
true);
+ int tableIndexNodeNum = buffer.getInt();
+ Map<String, MetadataIndexNode> tableIndexNodeMap = new HashMap<>();
+ for (int i = 0; i < tableIndexNodeNum; i++) {
+ String tableName = ReadWriteIOUtils.readString(buffer);
+ MetadataIndexNode metadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer, true);
+ tableIndexNodeMap.put(tableName, metadataIndexNode);
+ }
+ fileMetaData.setTableMetadataIndexNodeMap(tableIndexNodeMap);
+
+ // tableSchemas
+ int tableSchemaNum = buffer.getInt();
+ Map<String, TableSchema> tableSchemaMap = new HashMap<>();
+ for (int i = 0; i < tableSchemaNum; i++) {
+ String tableName = ReadWriteIOUtils.readString(buffer);
+ TableSchema tableSchema = TableSchema.deserialize(tableName, buffer);
+ tableSchemaMap.put(tableName, tableSchema);
+ }
+ fileMetaData.setTableSchemaMap(tableSchemaMap);
// metaOffset
long metaOffset = ReadWriteIOUtils.readLong(buffer);
@@ -87,9 +105,22 @@ public class TsFileMetadata {
public int serializeTo(OutputStream outputStream) throws IOException {
int byteLen = 0;
- // metadataIndex
- if (metadataIndex != null) {
- byteLen += metadataIndex.serializeTo(outputStream);
+ if (tableMetadataIndexNodeMap != null) {
+ byteLen += ReadWriteIOUtils.write(tableMetadataIndexNodeMap.size(),
outputStream);
+ for (Entry<String, MetadataIndexNode> entry :
tableMetadataIndexNodeMap.entrySet()) {
+ byteLen += ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ byteLen += entry.getValue().serializeTo(outputStream);
+ }
+ } else {
+ byteLen += ReadWriteIOUtils.write(0, outputStream);
+ }
+
+ if (tableSchemaMap != null) {
+ byteLen += ReadWriteIOUtils.write(tableSchemaMap.size(), outputStream);
+ for (Entry<String, TableSchema> entry : tableSchemaMap.entrySet()) {
+ byteLen += ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ byteLen += entry.getValue().serialize(outputStream);
+ }
} else {
byteLen += ReadWriteIOUtils.write(0, outputStream);
}
@@ -121,11 +152,12 @@ public class TsFileMetadata {
this.metaOffset = metaOffset;
}
- public MetadataIndexNode getMetadataIndex() {
- return metadataIndex;
+ public void setTableMetadataIndexNodeMap(
+ Map<String, MetadataIndexNode> tableMetadataIndexNodeMap) {
+ this.tableMetadataIndexNodeMap = tableMetadataIndexNodeMap;
}
- public void setMetadataIndex(MetadataIndexNode metadataIndex) {
- this.metadataIndex = metadataIndex;
+ public void setTableSchemaMap(Map<String, TableSchema> tableSchemaMap) {
+ this.tableSchemaMap = tableSchemaMap;
}
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/utils/CompatibilityUtils.java
b/tsfile/src/main/java/org/apache/tsfile/utils/CompatibilityUtils.java
new file mode 100644
index 00000000..4f8bcb99
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/tsfile/utils/CompatibilityUtils.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.utils;
+
+import org.apache.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.tsfile.file.metadata.TsFileMetadata;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+
+public class CompatibilityUtils {
+ public static TsFileMetadata deserializeTsFileMetadataFromV3(ByteBuffer
buffer) {
+ TsFileMetadata fileMetaData = new TsFileMetadata();
+
+ // metadataIndex
+ MetadataIndexNode metadataIndexNode =
MetadataIndexNode.deserializeFrom(buffer, true);
+ fileMetaData.setTableMetadataIndexNodeMap(Collections.singletonMap("",
metadataIndexNode));
+
+ // metaOffset
+ long metaOffset = ReadWriteIOUtils.readLong(buffer);
+ fileMetaData.setMetaOffset(metaOffset);
+
+ // read bloom filter
+ if (buffer.hasRemaining()) {
+ byte[] bytes =
ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(buffer);
+ int filterSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int hashFunctionSize =
ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ fileMetaData.setBloomFilter(
+ BloomFilter.buildBloomFilter(bytes, filterSize, hashFunctionSize));
+ }
+
+ return fileMetaData;
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/tsfile/write/schema/Schema.java
b/tsfile/src/main/java/org/apache/tsfile/write/schema/Schema.java
index 8894690d..091d14b4 100644
--- a/tsfile/src/main/java/org/apache/tsfile/write/schema/Schema.java
+++ b/tsfile/src/main/java/org/apache/tsfile/write/schema/Schema.java
@@ -128,4 +128,8 @@ public class Schema implements Serializable {
TableSchema tableSchema = tableSchemaMap.computeIfAbsent(tableName,
LogicalTableSchema::new);
tableSchema.update(chunkGroupMetadata);
}
+
+ public Map<String, TableSchema> getTableSchemaMap() {
+ return tableSchemaMap;
+ }
}
diff --git
a/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
b/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
index 57733141..289c92b9 100644
--- a/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
@@ -62,12 +62,14 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Queue;
import java.util.TreeMap;
import static
org.apache.tsfile.file.metadata.MetadataIndexConstructor.addCurrentIndexNodeToQueue;
import static
org.apache.tsfile.file.metadata.MetadataIndexConstructor.checkAndBuildLevelIndex;
import static
org.apache.tsfile.file.metadata.MetadataIndexConstructor.generateRootNode;
+import static
org.apache.tsfile.file.metadata.MetadataIndexConstructor.splitDeviceByTable;
/**
* TsFileIOWriter is used to construct metadata and write data stored in
memory to output stream.
@@ -451,10 +453,18 @@ public class TsFileIOWriter implements AutoCloseable {
measurementMetadataIndexQueue, out,
MetadataIndexNodeType.INTERNAL_MEASUREMENT));
}
- MetadataIndexNode metadataIndex =
checkAndBuildLevelIndex(deviceMetadataIndexMap, out);
+ Map<String, Map<IDeviceID, MetadataIndexNode>> tableDeviceNodesMap =
+ splitDeviceByTable(deviceMetadataIndexMap);
+
+ // build an index root for each table
+ Map<String, MetadataIndexNode> tableNodesMap = new HashMap<>();
+ for (Entry<String, Map<IDeviceID, MetadataIndexNode>> entry :
tableDeviceNodesMap.entrySet()) {
+ tableNodesMap.put(entry.getKey(),
checkAndBuildLevelIndex(entry.getValue(), out));
+ }
TsFileMetadata tsFileMetadata = new TsFileMetadata();
- tsFileMetadata.setMetadataIndex(metadataIndex);
+ tsFileMetadata.setTableMetadataIndexNodeMap(tableNodesMap);
+ tsFileMetadata.setTableSchemaMap(schema.getTableSchemaMap());
tsFileMetadata.setMetaOffset(metaOffset);
int size = tsFileMetadata.serializeTo(out.wrapAsStream());
