Re: [PR] Support schema region snapshot parser. [iotdb]

2024-03-14 Thread via GitHub 


SteveYurongSu merged PR #12145:
URL: https://github.com/apache/iotdb/pull/12145


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscr...@iotdb.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

Re: [PR] Support schema region snapshot parser. [iotdb]

2024-03-13 Thread via GitHub 


shuolinl commented on PR #12145:
URL: https://github.com/apache/iotdb/pull/12145#issuecomment-1996240419

   @SteveYurongSu Please take a look at this pr.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscr...@iotdb.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org

Re: [PR] Support schema region snapshot parser. [iotdb]

2024-03-10 Thread via GitHub 


Cpaulyz commented on code in PR #12145:
URL: https://github.com/apache/iotdb/pull/12145#discussion_r1519064617


##
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SchemaRegionSnapshotParser.java:
##
@@ -0,0 +1,501 @@
+package org.apache.iotdb.db.tools.schema;
+
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.file.SystemFileFactory;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.SchemaConstant;
+import org.apache.iotdb.commons.schema.node.IMNode;
+import org.apache.iotdb.commons.schema.node.common.AbstractDatabaseMNode;
+import org.apache.iotdb.commons.schema.node.common.AbstractMeasurementMNode;
+import org.apache.iotdb.commons.schema.node.utils.IMNodeContainer;
+import org.apache.iotdb.commons.schema.node.visitor.MNodeVisitor;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.queryengine.plan.statement.Statement;
+import 
org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateAlignedTimeSeriesStatement;
+import 
org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateTimeSeriesStatement;
+import 
org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ActivateTemplateStatement;
+import 
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode;
+import 
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.snapshot.MemMTreeSnapshotUtil.MNodeDeserializer;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import static org.apache.iotdb.commons.schema.SchemaConstant.ENTITY_MNODE_TYPE;
+import static 
org.apache.iotdb.commons.schema.SchemaConstant.INTERNAL_MNODE_TYPE;
+import static 
org.apache.iotdb.commons.schema.SchemaConstant.LOGICAL_VIEW_MNODE_TYPE;
+import static 
org.apache.iotdb.commons.schema.SchemaConstant.MEASUREMENT_MNODE_TYPE;
+import static 
org.apache.iotdb.commons.schema.SchemaConstant.STORAGE_GROUP_ENTITY_MNODE_TYPE;
+import static 
org.apache.iotdb.commons.schema.SchemaConstant.STORAGE_GROUP_MNODE_TYPE;
+import static 
org.apache.iotdb.commons.schema.SchemaConstant.isStorageGroupType;
+
+public class SchemaRegionSnapshotParser {
+
+  private static final Logger LOGGER = 
LoggerFactory.getLogger(SchemaRegionSnapshotParser.class);
+
+  private static StatementGener statementGener;
+
+  private static final IoTDBConfig CONFIG = 
IoTDBDescriptor.getInstance().getConfig();
+
+  private static final CommonConfig COMMON_CONFIG = 
CommonDescriptor.getInstance().getConfig();
+
+  public static List getSnapshotPaths() {
+String snapshotPath = CONFIG.getSchemaRegionConsensusDir();
+File snapshotDir = new File(snapshotPath);
+ArrayList schemaRegionList = new ArrayList<>();
+try (DirectoryStream stream = 
Files.newDirectoryStream(snapshotDir.toPath())) {
+  for (Path path : stream) {
+if (path.toFile().isDirectory()) {
+  schemaRegionList.add(path);
+}
+  }
+} catch (IOException exception) {
+  LOGGER.warn("cannot construct snapshot directory stream", exception);
+  return null;
+}
+if (schemaRegionList.isEmpty()) {
+  return null;
+}
+
+ArrayList latestSnapshots = new ArrayList<>();
+for (Path regionPath : schemaRegionList) {

Review Comment:
   Why not remove schemaRegionList and implement it in the previous loop?



##
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SchemaRegionSnapshotParser.java:
##
@@ -0,0 +1,501 @@
+package org.apache.iotdb.db.tools.schema;
+
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.file.SystemFileFactory;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.SchemaConstant;
+import org.apache.iotdb.commons.schema.node.IMNode;
+import org.apache.iotdb.commons.schema.node.common.AbstractDatabaseMNode;
+import org.apache.iotdb.commons.schema.node.common.AbstractMeasurementMNode;
+import org.apache.iotdb.commons.schema.node.utils.IMNodeContainer;
+import org.apache.iotdb.commons.schema.node.visitor.MNodeVisitor;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import

[PR] Support schema region snapshot parser. [iotdb]

2024-03-08 Thread via GitHub 


shuolinl opened a new pull request, #12145:
URL: https://github.com/apache/iotdb/pull/12145

   In this pr:
   1. Support translating snapshot file to statements, so you can use schema 
region snapshot parser to construct a same Schema tree in another cluster.
   2. Add some UTs.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscr...@iotdb.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org