This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 066fc35bc6 [IOTDB-3656] Loader Tool for TsFile (#6498)
066fc35bc6 is described below
commit 066fc35bc63505242425f9d3196850911dc76dc1
Author: yschengzi <[email protected]>
AuthorDate: Fri Jul 8 17:43:49 2022 +0800
[IOTDB-3656] Loader Tool for TsFile (#6498)
---
.../Maintenance-Tools/TsFile-Load-Tool.md | 62 +++
.../Maintenance-Tools/TsFile-Load-Tool.md | 62 +++
integration-test/pom.xml | 5 +
integration/pom.xml | 5 +
.../db/integration/IoTDBTsFileLoaderToolIT.java | 154 ++++++++
load-tsfile-tool/pom.xml | 83 ++++
load-tsfile-tool/src/assembly/loadTsFileTool.xml | 40 ++
.../src/assembly/resources/sbin/load-tsfile.bat | 60 +++
.../src/assembly/resources/sbin/load-tsfile.sh | 48 +++
.../java/org/apache/iotdb/TsFileLoaderTool.java | 427 +++++++++++++++++++++
pom.xml | 1 +
11 files changed, 947 insertions(+)
diff --git a/docs/UserGuide/Maintenance-Tools/TsFile-Load-Tool.md
b/docs/UserGuide/Maintenance-Tools/TsFile-Load-Tool.md
new file mode 100644
index 0000000000..83c906efa5
--- /dev/null
+++ b/docs/UserGuide/Maintenance-Tools/TsFile-Load-Tool.md
@@ -0,0 +1,62 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# IoTDB Load-TsFile Tool
+
+## Introduction
+
+The Load-TsFile tool is used to write the data in TsFile to the running IoTDB.
+
+## How to use
+
+Run load-tsfile.bat if you are in a Windows environment, or load-tsfile.sh if
you are on Linux or Unix.
+
+```bash
+./load-tsfile.bat -f filePath [-h host] [-help] [-p port] [-pw password] -u
user
+-f File or Dictionary to be loaded.
+-h Host Name (optional, default 127.0.0.1)
+-help Display help information(optional)
+-p Port (optional, default 6667)
+-pw password (optional)
+-u User name (required)
+```
+
+## Example
+
+Assuming that an IoTDB instance is running on server 192.168.0.101:6667, you
want to load all TsFile files from the locally saved TsFile backup folder
D:\IoTDB\data into this IoTDB instance.
+
+First move to the folder where load-tsfile.bat is located, open the command
line, and execute
+
+```bash
+./load-tsfile.bat -f "D:\IoTDB\data" -h 192.168.0.101 -p 6667 -u root -pw root
+```
+
+After waiting for the script execution to complete, you can check that the
data in the IoTDB instance has been loaded correctly.
+
+## Q&A
+
+- Cannot find or load the main class TsFileLoaderTool
+ - It may be because the environment variable $IOTDB_HOME is not set, please
set the environment variable and try again
+- Missing require argument: f or Missing require argument: u
+ - The input command is missing the -f field (file or folder path to be
loaded) or the -u field (user name), please add it and re-execute
+- What if the execution crashes in the middle and you want to reload?
+ - The easiest way, you re-execute the command just now, reloading the data
will not affect the correctness after loading
+ - If you want to save time by avoiding reloading a file that has already
been loaded, you can remove the TsFile that the last execution log shows has
been loaded from the pending folder and reload that folder
diff --git a/docs/zh/UserGuide/Maintenance-Tools/TsFile-Load-Tool.md
b/docs/zh/UserGuide/Maintenance-Tools/TsFile-Load-Tool.md
new file mode 100644
index 0000000000..472b9cb24b
--- /dev/null
+++ b/docs/zh/UserGuide/Maintenance-Tools/TsFile-Load-Tool.md
@@ -0,0 +1,62 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# IoTDB Load-TsFile Tool
+
+## 工具介绍
+
+Load-TsFile工具用于将TsFile中的数据写入正在运行的IoTDB中。
+
+## 使用方法
+
+若您在Windows环境中，请运行load-tsfile.bat，若为Linux或Unix，请运行load-tsfile.sh
+
+```bash
+./load-tsfile.bat -f filePath [-h host] [-help] [-p port] [-pw password] -u
user
+-f 待加载的文件或文件夹路径，必要字段
+-h IoTDB的Host地址，可选，默认127.0.0.1
+-help 输出帮助菜单，可选
+-p IoTDB的端口，可选，默认6667
+-pw IoTDB登录密码，可选，默认root
+-u IoTDb登录用户名，必要字段
+```
+
+## 使用范例
+
+假定服务器192.168.0.101:6667上运行一个IoTDB实例，想从将本地保存的TsFile备份文件夹D:\IoTDB\data中的所有的TsFile文件都加载进此IoTDB实例。
+
+首先移动到load-tsfile.bat所在文件夹中，打开命令行，然后执行
+
+```bash
+./load-tsfile.bat -f "D:\IoTDB\data" -h 192.168.0.101 -p 6667 -u root -pw root
+```
+
+等待脚本执行完成之后，可以检查IoTDB实例中数据已经被正确加载
+
+## 常见问题
+
+- 找不到或无法加载主类TsFileLoaderTool
+ - 可能是由于未设置环境变量$IOTDB_HOME，请设置环境变量之后重试
+- Missing require argument: f或Missing require argument: u
+ - 输入命令缺少待-f字段（加载文件或文件夹路径），或者缺少-u字段（用户名），请添加之后重新执行
+- 执行到中途崩溃了想重新加载怎么办
+ - 最简单的办法，您重新执行刚才的命令，重新加载数据不会影响加载之后的正确性
+ - 如果您想避免重新加载已经加载完成的文件来节省时间，您可以将上一次执行日志显示已经加载完成的TsFile从待加载文件夹中去掉，然后重新加载该文件夹
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index 37c81684f0..71a334ff1d 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -72,6 +72,11 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>load-tsfile-tool</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
<dependencyManagement>
<dependencies>
diff --git a/integration/pom.xml b/integration/pom.xml
index b502a63885..2a2617eda3 100644
--- a/integration/pom.xml
+++ b/integration/pom.xml
@@ -61,6 +61,11 @@
<version>${awaitility.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>load-tsfile-tool</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
<dependencyManagement>
<dependencies>
diff --git
a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBTsFileLoaderToolIT.java
b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBTsFileLoaderToolIT.java
new file mode 100644
index 0000000000..4e434ceb30
--- /dev/null
+++
b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBTsFileLoaderToolIT.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.TsFileLoaderTool;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+public class IoTDBTsFileLoaderToolIT {
+ private static final IoTDBConfig CONFIG =
IoTDBDescriptor.getInstance().getConfig();
+
+ private String tmpDir;
+
+ @Before
+ public void setup() throws Exception {
+ CONFIG.setEnableCrossSpaceCompaction(false);
+ CONFIG.setEnableSeqSpaceCompaction(false);
+ CONFIG.setEnableUnseqSpaceCompaction(false);
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ }
+
+ public void prepareTsFiles() throws Exception {
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("set storage group to root.sg");
+ statement.execute("create timeseries root.sg.d1.s INT32");
+ statement.execute("create timeseries root.sg.d2.s INT32");
+ statement.execute("create timeseries root.sg.d3.s INT32");
+
+ statement.execute("insert into root.sg.d1(timestamp, s) values(1, 1),
(2, 2)");
+ statement.execute("insert into root.sg.d2(timestamp, s) values(3, 3),
(4, 4)");
+ statement.execute("insert into root.sg.d3(timestamp, s) values(5, 5),
(6, 6)");
+ statement.execute("flush");
+
+ statement.execute("insert into root.sg.d1(timestamp, s) values(3, 3),
(4, 4)");
+ statement.execute("insert into root.sg.d2(timestamp, s) values(1, 1),
(2, 2)");
+ statement.execute("insert into root.sg.d3(timestamp, s) values(1, 1),
(2, 2)");
+ statement.execute("flush");
+
+ statement.execute("insert into root.sg.d1(timestamp, s) values(5, 5),
(6, 6)");
+ statement.execute("insert into root.sg.d2(timestamp, s) values(5, 5),
(6, 6)");
+ statement.execute("insert into root.sg.d3(timestamp, s) values(3, 3),
(4, 4)");
+ statement.execute("flush");
+
+ statement.execute("delete from root.sg.d1.s where timestamp <= 2");
+
+ statement.execute("create aligned timeseries root.sg.a(s1 INT32, s2
TEXT)");
+ statement.execute("insert into root.sg.a(time, s1, s2) aligned values(1,
1, '1')");
+ statement.execute("insert into root.sg.a(time, s1, s2) aligned values(2,
2, '2')");
+ statement.execute("insert into root.sg.a(time, s2) aligned values(3,
'3')");
+ statement.execute("insert into root.sg.a(time, s1, s2) aligned values(4,
4, '3')");
+ statement.execute("flush");
+
+ statement.execute("delete from root.sg.a.s1 where time > 2");
+
+ for (TsFileResource resource :
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.sg"))
+ .getSequenceFileList()) {
+ if (tmpDir == null) {
+ tmpDir =
+ resource
+ .getTsFile()
+ .getParentFile()
+ .getParentFile()
+ .getParentFile()
+ .getParentFile()
+ .getParent()
+ + File.separator
+ + "tmp";
+ File tmpFile = new File(tmpDir);
+ if (!tmpFile.exists()) {
+ tmpFile.mkdirs();
+ }
+ }
+ statement.execute(String.format("unload '%s' '%s'",
resource.getTsFilePath(), tmpDir));
+ }
+ for (TsFileResource resource :
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.sg"))
+ .getUnSequenceFileList()) {
+ statement.execute(String.format("unload '%s' '%s'",
resource.getTsFilePath(), tmpDir));
+ }
+ }
+ }
+
+ private void checkRes(ResultSet resultSet, int start, int end) throws
Exception {
+ while (resultSet.next()) {
+ Assert.assertEquals(start, resultSet.getInt(2));
+ start += 1;
+ }
+ Assert.assertEquals(end, start - 1);
+ }
+
+ @Test
+ public void testLoadTsFile() {
+ try {
+ prepareTsFiles();
+
+ String[] args = {"-h", "127.0.0.1", "-p", "6667", "-u", "root", "-pw",
"root", "-f", tmpDir};
+ TsFileLoaderTool.main(args);
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667",
"root", "root");
+ Statement statement = connection.createStatement()) {
+ checkRes(statement.executeQuery("select * from root.sg.d1"), 3, 6);
+ checkRes(statement.executeQuery("select * from root.sg.d2"), 1, 6);
+ checkRes(statement.executeQuery("select * from root.sg.d3"), 1, 6);
+ checkRes(statement.executeQuery("select s1 from root.sg.a"), 1, 2);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+}
diff --git a/load-tsfile-tool/pom.xml b/load-tsfile-tool/pom.xml
new file mode 100644
index 0000000000..9d526d4f8d
--- /dev/null
+++ b/load-tsfile-tool/pom.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>iotdb-parent</artifactId>
+ <groupId>org.apache.iotdb</groupId>
+ <version>0.14.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>load-tsfile-tool</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-server</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-session</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <executions>
+ <!-- Package binaries-->
+ <execution>
+ <id>client-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+
<descriptor>src/assembly/loadTsFileTool.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <archive>
+ <manifest>
+
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+ </manifest>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
+</project>
diff --git a/load-tsfile-tool/src/assembly/loadTsFileTool.xml
b/load-tsfile-tool/src/assembly/loadTsFileTool.xml
new file mode 100644
index 0000000000..68b7510885
--- /dev/null
+++ b/load-tsfile-tool/src/assembly/loadTsFileTool.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly>
+ <id>RewriteFileTool</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>src/assembly/resources</directory>
+ <outputDirectory>${file.separator}</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/load-tsfile-tool/src/assembly/resources/sbin/load-tsfile.bat
b/load-tsfile-tool/src/assembly/resources/sbin/load-tsfile.bat
new file mode 100644
index 0000000000..9972b449f6
--- /dev/null
+++ b/load-tsfile-tool/src/assembly/resources/sbin/load-tsfile.bat
@@ -0,0 +1,60 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+echo ````````````````````````````````````````````````
+echo Starting IoTDB Client Import Script
+echo ````````````````````````````````````````````````
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.TsFileLoaderTool
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM
-----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+set JAVA_OPTS=-ea^
+ -DIOTDB_HOME=%IOTDB_HOME%
+
+@REM ***** CLASSPATH library setting *****
+set CLASSPATH=%IOTDB_HOME%\lib\*
+
+REM
-----------------------------------------------------------------------------
+
+"%JAVA_HOME%\bin\java" -DIOTDB_HOME=%IOTDB_HOME% %JAVA_OPTS% -cp %CLASSPATH%
%MAIN_CLASS% %*
+set ret_code=%ERRORLEVEL%
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+set ret_code=1
+pause
+
+
+@REM
-----------------------------------------------------------------------------
+:finally
+
+ENDLOCAL
+
+EXIT /B %ret_code%
diff --git a/load-tsfile-tool/src/assembly/resources/sbin/load-tsfile.sh
b/load-tsfile-tool/src/assembly/resources/sbin/load-tsfile.sh
new file mode 100644
index 0000000000..038a4c9c0d
--- /dev/null
+++ b/load-tsfile-tool/src/assembly/resources/sbin/load-tsfile.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ---------------------
+echo Starting Loading TsFiles
+echo ---------------------
+
+if [ -z "${IOTDB_HOME}" ]; then
+ export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}":"$f
+done
+
+MAIN_CLASS=org.apache.iotdb.TsFileLoaderTool
+
+"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?
diff --git
a/load-tsfile-tool/src/main/java/org/apache/iotdb/TsFileLoaderTool.java
b/load-tsfile-tool/src/main/java/org/apache/iotdb/TsFileLoaderTool.java
new file mode 100644
index 0000000000..8979fa6f1f
--- /dev/null
+++ b/load-tsfile-tool/src/main/java/org/apache/iotdb/TsFileLoaderTool.java
@@ -0,0 +1,427 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.modification.ModificationFile;
+import org.apache.iotdb.db.utils.QueryUtils;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+import org.apache.iotdb.tsfile.exception.write.NoMeasurementException;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Field;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.common.RowRecord;
+import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
+import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier;
+import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
+import org.apache.iotdb.tsfile.read.query.dataset.DataSetWithoutTimeGenerator;
+import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.iotdb.tsfile.read.reader.series.AbstractFileSeriesReader;
+import org.apache.iotdb.tsfile.read.reader.series.EmptyFileSeriesReader;
+import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
+import org.apache.iotdb.tsfile.utils.FilePathUtils;
+import org.apache.iotdb.tsfile.write.record.Tablet;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class TsFileLoaderTool {
+ private static final int MAX_TABLET_LENGTH = 1024 * 64;
+
+ private static String host = "localhost";
+ private static String port = "6667";
+ private static String user = "root";
+ private static String password = "root";
+ private static String filePath = "";
+
+ private static Map<String, Set<MeasurementSchema>> device2Measurements;
+
+ public static void main(String[] args) {
+ Session session = null;
+ try {
+ parseArgs(args);
+ session = new Session(host, port, user, password);
+ session.open();
+ System.out.printf("Connect to IoTDB %s:%s successfully.%n", host, port);
+ writeToIoTDB(collectTsFiles(new File(filePath)), session);
+ } catch (IoTDBConnectionException e) {
+ System.out.printf("Can not connect to IoTDB. %s%n", e.getMessage());
+ e.printStackTrace();
+ } catch (Exception e) {
+ System.out.printf("Load Error. %s%n", e.getMessage());
+ e.printStackTrace();
+ } finally {
+ if (session != null) {
+ try {
+ session.close();
+ } catch (IoTDBConnectionException e) {
+ System.out.printf("Can not connect to IoTDB. %s%n", e.getMessage());
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ public static void parseArgs(String[] args) {
+ Options options = createOptions();
+ try {
+ CommandLine commandLine = new DefaultParser().parse(options, args);
+ host = getArgOrDefault(commandLine, "h", host);
+ port = getArgOrDefault(commandLine, "p", port);
+ user = getArgOrDefault(commandLine, "u", user);
+ password = getArgOrDefault(commandLine, "pw", password);
+ filePath = getArgOrDefault(commandLine, "f", filePath);
+ } catch (ParseException e) {
+ System.out.printf("Parse Args Error. %s%n", e.getMessage());
+ priHelp(options);
+ }
+ }
+
+ private static void priHelp(Options options) {
+ new HelpFormatter().printHelp("./load-tsfile.sh(load-tsfile.bat if
Windows)", options, true);
+ }
+
+ private static String getArgOrDefault(
+ CommandLine commandLine, String argName, String defaultValue) {
+ String value = commandLine.getOptionValue(argName);
+ return value == null ? defaultValue : value;
+ }
+
+ public static Options createOptions() {
+ Options options = new Options();
+ Option help = new Option("help", false, "Display help
information(optional)");
+ help.setRequired(false);
+ options.addOption(help);
+
+ Option host =
+ Option.builder("h")
+ .argName("host")
+ .hasArg()
+ .desc("Host Name (optional, default 127.0.0.1)")
+ .build();
+ options.addOption(host);
+
+ Option port =
+ Option.builder("p").argName("port").hasArg().desc("Port (optional,
default 6667)").build();
+ options.addOption(port);
+
+ Option username =
+ Option.builder("u")
+ .argName("username")
+ .hasArg()
+ .desc("User name (required)")
+ .required()
+ .build();
+ options.addOption(username);
+
+ Option password =
+ Option.builder("pw").argName("password").hasArg().desc("password
(optional)").build();
+ options.addOption(password);
+
+ Option filePathOpt =
+ Option.builder("f")
+ .argName("file")
+ .hasArg()
+ .desc("File or Dictionary to be loaded.")
+ .required()
+ .build();
+ options.addOption(filePathOpt);
+ return options;
+ }
+
+ public static List<File> collectTsFiles(File file) {
+ if (file.isFile()) {
+ return file.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)
+ ? Collections.singletonList(file)
+ : Collections.emptyList();
+ }
+ List<File> list = new ArrayList<>();
+ for (File listFile : file.listFiles()) {
+ if (listFile.isDirectory()) {
+ list.addAll(collectTsFiles(listFile));
+ } else if (listFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
+ list.add(listFile);
+ }
+ }
+ return list;
+ }
+
+ /**
+ * write a list of file to IoTDB with session.
+ *
+ * @param files a list of file to write to IoTDB
+ * @param session IoTDB session
+ */
+ public static void writeToIoTDB(List<File> files, Session session) {
+ sortTsFiles(files);
+ int size = files.size();
+ List<File> unloadTsFiles = new ArrayList<>();
+ System.out.printf("Collect TsFiles successfully, %d files to be
loaded.%n", size);
+ System.out.println("Start Loading TsFiles...");
+ for (int i = 0; i < size; i++) {
+ File file = files.get(i);
+ System.out.printf("Loading %s(%d/%d)...", file.getPath(), i + 1, size);
+ try {
+ writeTsFile(file.getPath(), session);
+ } catch (Exception e) {
+ System.out.println(
+ "------------------------------Error
Message------------------------------");
+ e.printStackTrace();
+ System.out.println(
+ "------------------------------End
Message------------------------------");
+ unloadTsFiles.add(file);
+ continue;
+ }
+ System.out.println("Done");
+ }
+ System.out.println("Finish Loading TsFiles");
+ System.out.printf(
+ "Load %d TsFiles successfully, %d TsFiles not loaded.%n",
+ size - unloadTsFiles.size(), unloadTsFiles.size());
+ if (!unloadTsFiles.isEmpty()) {
+ System.out.println("Load Error TsFiles list");
+ for (File file : unloadTsFiles) {
+ System.out.println(file.getPath());
+ }
+ }
+ }
+
+ private static void sortTsFiles(List<File> files) {
+ Map<File, Long> file2Timestamp = new HashMap<>();
+ Map<File, Long> file2Version = new HashMap<>();
+ for (File file : files) {
+ String[] splitStrings =
file.getName().split(FilePathUtils.FILE_NAME_SEPARATOR);
+ file2Timestamp.put(file, Long.parseLong(splitStrings[0]));
+ file2Version.put(file, Long.parseLong(splitStrings[1]));
+ }
+
+ Collections.sort(
+ files,
+ (o1, o2) -> {
+ long timestampDiff = file2Timestamp.get(o1) - file2Timestamp.get(o2);
+ if (timestampDiff != 0) {
+ return (int) (timestampDiff);
+ }
+ return (int) (file2Version.get(o1) - file2Version.get(o2));
+ });
+ }
+
+ /**
+ * Read a TsFile and write into IoTDB session. This method can load TsFile
with IoTDB version.
+ * Support TsFile generated from IoTDB version 0.12 - 0.14(including Aligned
Timeseries).
+ *
+ * @param filename the file path to be loaded
+ * @param session IoTDB session
+ */
+ public static void writeTsFile(String filename, Session session)
+ throws IOException, IllegalPathException, IoTDBConnectionException,
+ StatementExecutionException, NoMeasurementException {
+ // parse modifications from .mods
+ List<Modification> modifications = null;
+ if (FSFactoryProducer.getFSFactory()
+ .getFile(filename + ModificationFile.FILE_SUFFIX)
+ .exists()) {
+ modifications =
+ (List<Modification>)
+ new ModificationFile(filename +
ModificationFile.FILE_SUFFIX).getModifications();
+ }
+
+ // read all device and their measurements
+ parseDeviceFromTsFile(filename);
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(filename)) {
+ for (Map.Entry<String, Set<MeasurementSchema>> entry :
device2Measurements.entrySet()) {
+ // collect measurements for device
+ boolean isAligned = false;
+ String curDevice = entry.getKey();
+ List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+ ArrayList<Path> paths = new ArrayList<>();
+ for (MeasurementSchema measurementSchema : entry.getValue()) {
+ if (!measurementSchema.getType().equals(TSDataType.VECTOR)) {
+ measurementSchemas.add(measurementSchema);
+ } else {
+ isAligned = true;
+ }
+ }
+ for (MeasurementSchema measurementSchema : measurementSchemas) {
+ paths.add(new Path(curDevice, measurementSchema.getMeasurementId()));
+ }
+
+ // construct query to this tsfile
+ List<AbstractFileSeriesReader> readersOfSelectedSeries = new
ArrayList<>();
+ List<TSDataType> dataTypes = new ArrayList<>();
+ IMetadataQuerier metadataQuerier = new
MetadataQuerierByFileImpl(reader);
+ IChunkLoader chunkLoader = new CachedChunkLoaderImpl(reader);
+ for (Path path : paths) {
+ List<IChunkMetadata> chunkMetadataList =
metadataQuerier.getChunkMetaDataList(path);
+ modifyChunkMetadata(isAligned, path, chunkMetadataList,
modifications);
+ AbstractFileSeriesReader seriesReader;
+ if (chunkMetadataList.isEmpty()) {
+ seriesReader = new EmptyFileSeriesReader();
+ dataTypes.add(metadataQuerier.getDataType(path));
+ } else {
+ seriesReader = new FileSeriesReader(chunkLoader,
chunkMetadataList, null);
+ dataTypes.add(chunkMetadataList.get(0).getDataType());
+ }
+ readersOfSelectedSeries.add(seriesReader);
+ }
+
+ // read data from tsfile and construct session to send to IoTDB
+ QueryDataSet dataSet =
+ new DataSetWithoutTimeGenerator(paths, dataTypes,
readersOfSelectedSeries);
+ Tablet tablet = new Tablet(curDevice, measurementSchemas,
MAX_TABLET_LENGTH);
+ tablet.initBitMaps();
+ int measurementSize = measurementSchemas.size();
+ while (dataSet.hasNext()) {
+ RowRecord rowRecord = dataSet.next();
+ tablet.addTimestamp(tablet.rowSize, rowRecord.getTimestamp());
+ for (int i = 0; i < measurementSize; i++) {
+ Field field = rowRecord.getFields().get(i);
+ if (field == null) {
+ tablet.bitMaps[i].mark(tablet.rowSize);
+ } else {
+ tablet.addValue(
+ measurementSchemas.get(i).getMeasurementId(),
+ tablet.rowSize,
+ field.getObjectValue(field.getDataType()));
+ }
+ }
+ tablet.rowSize++;
+ if (tablet.rowSize == MAX_TABLET_LENGTH) {
+ if (isAligned) {
+ session.insertAlignedTablet(tablet);
+ } else {
+ session.insertTablet(tablet);
+ }
+ tablet.reset();
+ }
+ }
+ if (isAligned) {
+ session.insertAlignedTablet(tablet);
+ } else {
+ session.insertTablet(tablet);
+ }
+ }
+ }
+ }
+
+ private static void parseDeviceFromTsFile(String filename) throws
IOException {
+ device2Measurements = new HashMap<>();
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(filename)) {
+ reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1);
+ String curDevice = null;
+ byte marker;
+ while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) {
+ switch (marker) {
+ case MetaMarker.CHUNK_HEADER:
+ case MetaMarker.TIME_CHUNK_HEADER:
+ case MetaMarker.VALUE_CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER:
+ ChunkHeader header = reader.readChunkHeader(marker);
+ MeasurementSchema measurementSchema =
+ new MeasurementSchema(
+ header.getMeasurementID(),
+ header.getDataType(),
+ header.getEncodingType(),
+ header.getCompressionType());
+ device2Measurements
+ .computeIfAbsent(curDevice, o -> new HashSet<>())
+ .add(measurementSchema);
+ reader.position(reader.position() + header.getDataSize());
+ break;
+ case MetaMarker.CHUNK_GROUP_HEADER:
+ ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader();
+ curDevice = chunkGroupHeader.getDeviceID();
+ break;
+ case MetaMarker.OPERATION_INDEX_RANGE:
+ reader.readPlanIndex();
+ break;
+ default:
+ MetaMarker.handleUnexpectedMarker(marker);
+ }
+ }
+ }
+ }
+
+ private static void modifyChunkMetadata(
+ boolean isAligned,
+ Path path,
+ List<IChunkMetadata> chunkMetadataList,
+ List<Modification> modifications)
+ throws IllegalPathException {
+ if (modifications == null || modifications.isEmpty()) {
+ return;
+ }
+ List<Modification> measurementModifications = new ArrayList<>();
+ Iterator<Modification> modsIterator = modifications.listIterator();
+ Deletion currentDeletion;
+ while (modsIterator.hasNext()) {
+ currentDeletion = (Deletion) modsIterator.next();
+ // if deletion path match the chunkPath, then add the deletion to the
list
+ if (currentDeletion.getPath().matchFullPath(new
PartialPath(path.getFullPath()))) {
+ measurementModifications.add(currentDeletion);
+ }
+ }
+ if (!isAligned) {
+ QueryUtils.modifyChunkMetaData(chunkMetadataList,
measurementModifications);
+ } else {
+ List<AlignedChunkMetadata> alignedChunkMetadataList = new ArrayList<>();
+ for (IChunkMetadata chunkMetadata : chunkMetadataList) {
+ alignedChunkMetadataList.add((AlignedChunkMetadata) chunkMetadata);
+ }
+ // AlignedChunk only contains one valueChunkMetadata which is
measurement with this path
+ QueryUtils.modifyAlignedChunkMetaData(
+ alignedChunkMetadataList,
Collections.singletonList(measurementModifications));
+ }
+ }
+}
diff --git a/pom.xml b/pom.xml
index ba7281a7d3..cebab7c6db 100644
--- a/pom.xml
+++ b/pom.xml
@@ -119,6 +119,7 @@
<module>library-udf</module>
<module>schema-engine-rocksdb</module>
<module>udf-api</module>
+ <module>load-tsfile-tool</module>
</modules>
<!-- Properties Management -->
<properties>
