This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new b5ec1d9f Fixed testing bug (#201)
b5ec1d9f is described below
commit b5ec1d9fe36c8cd2099c79093fccc7f6136f5a9c
Author: CloudWise-Lukemiao <[email protected]>
AuthorDate: Fri Aug 23 10:24:26 2024 +0800
Fixed testing bug (#201)
* Fixed testing bug
* Fixed testing bug
* Fixed ut failed
* Fixed ut failed
* Optimize code based on review comments
* Optimize code based on review comments
* Optimize code based on review comments
* Optimize code based on review comments
* Optimize code based on review comments
---------
Co-authored-by: luke.miao <[email protected]>
---
java/tools/pom.xml | 18 +++
.../assembly/resources/conf/logback-cvs2tsfile.xml | 5 +-
.../src/assembly/resources/tools/csv2tsfile.bat | 5 +-
java/tools/src/assembly/tools.xml | 3 +-
.../java/org/apache/tsfile/tools/SchemaParser.java | 38 ++++--
.../java/org/apache/tsfile/tools/TsFileTool.java | 140 +++++++++++++++++----
.../org/apache/tsfile/tools/TsfiletoolsTest.java | 75 ++++++++++-
7 files changed, 240 insertions(+), 44 deletions(-)
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index d46e695c..e7e9de18 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -43,11 +43,19 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile</artifactId>
<version>1.2.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@@ -60,6 +68,16 @@
</dependencies>
<build>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <usedDependencies combine.children="append">
+ <!-- We just need this dependency to prevent the
compiler from freaking out on unused imports -->
+
<usedDependency>ch.qos.logback:logback-classic</usedDependency>
+ </usedDependencies>
+ </configuration>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
diff --git a/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml
b/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml
index d4d2ed8f..84a56a6c 100644
--- a/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml
+++ b/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml
@@ -16,6 +16,7 @@
under the License.
-->
<configuration scan="true" scanPeriod="60 seconds">
+ <statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
<appender class="ch.qos.logback.core.ConsoleAppender" name="stdout">
<Target>System.out</Target>
<encoder>
@@ -23,7 +24,7 @@
<charset>utf-8</charset>
</encoder>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>ERROR</level>
+ <level>INFO</level>
</filter>
</appender>
<appender class="ch.qos.logback.core.rolling.RollingFileAppender"
name="tools">
@@ -46,4 +47,6 @@
<appender-ref ref="stdout"/>
<appender-ref ref="tools"/>
</root>
+ <logger name="org.apache.tsfile.common.conf.TSFileDescriptor" level="OFF"/>
+ <logger name="org.apache.tsfile.write.TsFileWriter" level="OFF"/>
</configuration>
diff --git a/java/tools/src/assembly/resources/tools/csv2tsfile.bat
b/java/tools/src/assembly/resources/tools/csv2tsfile.bat
index d8bc1dff..ec03e3e1 100644
--- a/java/tools/src/assembly/resources/tools/csv2tsfile.bat
+++ b/java/tools/src/assembly/resources/tools/csv2tsfile.bat
@@ -39,11 +39,10 @@ set CLASSPATH="%TSFILE_HOME%\lib\*"
if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.tsfile.tools.TsFileTool
set TSFILE_CONF=%TSFILE_HOME%\conf
-set
"tsfile_params=-Dlogback.configurationFile=!IOTDB_CLI_CONF!\logback-cvs2tsfile.xml"
-start /B "" cmd /C "("%JAVA_HOME%\bin\java" -DTSFILE_HOME=!TSFILE_HOME!
!tsfile_params! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*) > nul 2>&1"
+set
"tsfile_params=-Dlogback.configurationFile=!TSFILE_CONF!\logback-cvs2tsfile.xml"
+start /B /WAIT "" cmd /C "("%JAVA_HOME%\bin\java" -DTSFILE_HOME=!TSFILE_HOME!
!tsfile_params! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*)"
exit /b
-
:err
echo JAVA_HOME environment variable must be set!
set ret_code=1
diff --git a/java/tools/src/assembly/tools.xml
b/java/tools/src/assembly/tools.xml
index ccd96051..d03bca13 100644
--- a/java/tools/src/assembly/tools.xml
+++ b/java/tools/src/assembly/tools.xml
@@ -25,7 +25,7 @@
<format>dir</format>
<format>zip</format>
</formats>
- <includeBaseDirectory>false</includeBaseDirectory>
+ <includeBaseDirectory>true</includeBaseDirectory>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
@@ -45,6 +45,7 @@
<file>
<source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/csv2tsfile.sh</source>
<destName>tools/csv2tsfile.sh</destName>
+ <fileMode>0755</fileMode>
</file>
<file>
<source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/csv2tsfile.bat</source>
diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java
b/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java
index 0bb9816e..779cfbb1 100644
--- a/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java
+++ b/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java
@@ -23,16 +23,17 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
public class SchemaParser {
public static class Schema {
- String tableName;
- String timePrecision;
+ String tableName = "";
+ String timePrecision = "ms";
boolean hasHeader = true;
- String separator;
+ String separator = ",";
String nullFormat;
- String timeColumn;
+ String timeColumn = "";
int timeColumnIndex = -1;
List<IDColumns> idColumns = new ArrayList<>();
List<Column> csvColumns = new ArrayList<>();
@@ -155,7 +156,12 @@ public class SchemaParser {
} else if (line.startsWith("time_precision=")) {
schema.timePrecision = extractValue(line);
} else if (line.startsWith("has_header=")) {
- schema.hasHeader = Boolean.parseBoolean(extractValue(line));
+ String has_header = extractValue(line);
+ if (has_header.equals("true") || has_header.equals("false")) {
+ schema.hasHeader = Boolean.parseBoolean(has_header);
+ } else {
+ throw new IllegalArgumentException("The data format of has_header
is incorrect");
+ }
} else if (line.startsWith("separator=")) {
schema.separator = extractValue(line);
} else if (line.startsWith("null_format=")) {
@@ -208,7 +214,7 @@ public class SchemaParser {
for (IDColumns idColumn : idColumnsList) {
if (!idColumn.isDefault) {
for (int j = 0; j < columnList.size(); j++) {
- if (columnList.get(j).name.equals(idColumn.name)) {
+ if (Objects.equals(columnList.get(j).name, idColumn.name)) {
idColumn.csvColumnIndex = j;
break;
}
@@ -244,21 +250,33 @@ public class SchemaParser {
if (!schema.timePrecision.equals("us")
&& !schema.timePrecision.equals("ms")
&& !schema.timePrecision.equals("ns")) {
- throw new IllegalArgumentException("timePrecision must be us,ms or ns");
+ throw new IllegalArgumentException("The time_precision parameter only
supports ms,us,ns");
}
if (!schema.separator.equals(",")
&& !schema.separator.equals("tab")
&& !schema.separator.equals(";")) {
throw new IllegalArgumentException("separator must be \",\", tab, or
\";\"");
}
- if (schema.timeColumnIndex < 0) {
- throw new IllegalArgumentException("time_column is required");
- }
if (schema.tableName.isEmpty()) {
throw new IllegalArgumentException("table_name is required");
}
+ if (schema.idColumns.isEmpty()) {
+ throw new IllegalArgumentException("id_columns is required");
+ }
if (schema.csvColumns.isEmpty()) {
throw new IllegalArgumentException("csv_columns is required");
}
+ if (schema.timeColumn.isEmpty()) {
+ throw new IllegalArgumentException("time_column is required");
+ } else if (schema.timeColumnIndex < 0) {
+ throw new IllegalArgumentException(
+ "The value " + schema.timeColumn + " of time_column is not in
csv_columns");
+ }
+ for (IDColumns idColumn : schema.idColumns) {
+ if (idColumn.csvColumnIndex < 0 && !idColumn.isDefault) {
+ throw new IllegalArgumentException(
+ "The value " + idColumn.name + " of id_columns is not in
csv_columns");
+ }
+ }
}
}
diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java
b/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java
index 0d5c1cfe..2c62982e 100644
--- a/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java
+++ b/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java
@@ -35,11 +35,14 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
@@ -97,6 +100,7 @@ public class TsFileTool {
} catch (InterruptedException e) {
LOGGER.error("Failed to await termination", e);
}
+ LOGGER.info("The " + inputDirectoryStr + " directory or file has
completed execution");
}
}
@@ -140,8 +144,14 @@ public class TsFileTool {
return new TableSchema(tableName, measurementSchemas, columnTypes);
}
- private static boolean writeTsFile(String fileName, List<String> lineList) {
- final File tsFile = new File(outputDirectoryStr, fileName);
+ private static boolean writeTsFile(
+ String sourceFilePath, String fileName, List<String> lineList) {
+ String inputFileAbsolutePath = new
File(inputDirectoryStr).getAbsolutePath();
+ String soureFlieName = new File(sourceFilePath).getName();
+ String fileOutPutDirStr =
+ outputDirectoryStr
+ + sourceFilePath.replace(inputFileAbsolutePath,
"").replace(soureFlieName, "");
+ final File tsFile = new File(fileOutPutDirStr, fileName);
TsFileWriter writer = null;
try {
writer = new TsFileWriter(tsFile);
@@ -158,20 +168,25 @@ public class TsFileTool {
return false;
}
} catch (Exception e) {
- e.printStackTrace();
- LOGGER.error("Failed to write file: " + tsFile);
+ LOGGER.error("Failed to write file: " + tsFile.getAbsolutePath(), e);
return false;
} finally {
if (writer != null) {
try {
writer.close();
} catch (IOException e) {
- e.printStackTrace();
+ LOGGER.error("Failed to close file: " + tsFile.getAbsolutePath(), e);
}
}
}
}
+ private static void deleteFile(File tsfile) {
+ if (!tsfile.delete()) {
+ LOGGER.error(tsfile.getAbsolutePath() + " delete failed");
+ }
+ }
+
private static Tablet genTablet(
TableSchema tableSchema, List<String> lineList, Map<String, Object>
defaultMap) {
int num = lineList.size();
@@ -188,9 +203,9 @@ public class TsFileTool {
map.put(column.name, i);
}
try {
+ List<String[]> parsedLines = sortAndParseLines(lineList);
for (int i = 0; i < num; i++) {
- String line = lineList.get(i);
- String[] lineArray = line.split(schema.separator);
+ String[] lineArray = parsedLines.get(i);
long timestamp =
DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
lineArray[schema.timeColumnIndex], schema.timePrecision);
@@ -219,11 +234,30 @@ public class TsFileTool {
tablet.rowSize = num;
return tablet;
} catch (Exception e) {
- LOGGER.error("Failed to parse csv file");
+ LOGGER.error("Failed to parse csv file", e);
}
return null;
}
+ public static List<String[]> sortAndParseLines(List<String> data) {
+ List<String[]> parsedLines = new ArrayList<>(data.size());
+
+ for (String line : data) {
+ parsedLines.add(line.split(schema.separator));
+ }
+ parsedLines.sort(
+ (o1, o2) -> {
+ long time1 =
+ DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
+ o1[schema.timeColumnIndex], schema.timePrecision);
+ long time2 =
+ DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
+ o2[schema.timeColumnIndex], schema.timePrecision);
+ return Long.compare(time1, time2);
+ });
+ return parsedLines;
+ }
+
public static Object getValue(TSDataType dataType, String i,
Tablet.ColumnType columnType) {
switch (dataType) {
case INT64:
@@ -266,15 +300,46 @@ public class TsFileTool {
private static void cpFile(String sourceFilePath, String
targetDirectoryPath) {
try {
- Files.createDirectories(Paths.get(targetDirectoryPath));
+ String inputFileAbsolutePath = new
File(inputDirectoryStr).getAbsolutePath();
+ String soureFlieName = new File(sourceFilePath).getName();
+ String fileOutPutDirStr =
+ targetDirectoryPath
+ + sourceFilePath.replace(inputFileAbsolutePath,
"").replace(soureFlieName, "");
+ Files.createDirectories(Paths.get(fileOutPutDirStr));
Path sourcePath = Paths.get(sourceFilePath);
- Path targetPath = Paths.get(targetDirectoryPath,
sourcePath.getFileName().toString());
+ Path targetPath = Paths.get(fileOutPutDirStr,
sourcePath.getFileName().toString());
Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
LOGGER.error("Failed to copy file: " + sourceFilePath, e);
}
}
+ public static void writeToNewCSV(
+ String headerLine, String fileAbsolutePath, List<String> data, String
newFileName) {
+ if (schema.hasHeader && StringUtils.isNotEmpty(headerLine)) {
+ data.add(0, headerLine);
+ }
+ String inputFileAbsolutePath = new
File(inputDirectoryStr).getAbsolutePath();
+ String soureFlieName = new File(fileAbsolutePath).getName();
+ String fileOutPutDirStr =
+ failedDirectoryStr
+ + fileAbsolutePath.replace(inputFileAbsolutePath,
"").replace(soureFlieName, "");
+ try {
+ Files.createDirectories(Paths.get(fileOutPutDirStr));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ String path = Paths.get(fileOutPutDirStr,
newFileName).toFile().getAbsolutePath();
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(path))) {
+ for (String line : data) {
+ writer.write(line);
+ writer.newLine();
+ }
+ } catch (IOException e) {
+ LOGGER.error("Error writing to CSV file", e);
+ }
+ }
+
private static void processFile(File inputFile, ExecutorService executor) {
AtomicInteger fileCounter = new AtomicInteger(1);
String fileName = FilenameUtils.getBaseName(inputFile.getName());
@@ -289,6 +354,7 @@ public class TsFileTool {
int index = 0;
List<String> lineList = new ArrayList<>();
boolean isSingleFile = true;
+ String headerLine = null;
while ((line = reader.readLine()) != null) {
if (index == 0) {
if (schema.timeColumnIndex == -1) {
@@ -308,6 +374,7 @@ public class TsFileTool {
}
if (schema.hasHeader && index == 0) {
+ headerLine = line;
index++;
continue;
}
@@ -318,6 +385,7 @@ public class TsFileTool {
isSingleFile = false;
if (chunkLines > 0) {
submitChunk(
+ headerLine,
lineList,
fileCounter.getAndIncrement(),
executor,
@@ -330,6 +398,7 @@ public class TsFileTool {
} else {
lineList.add(line);
submitChunk(
+ headerLine,
lineList,
fileCounter.getAndIncrement(),
executor,
@@ -347,6 +416,7 @@ public class TsFileTool {
}
if (lineList.size() > 0) {
submitChunk(
+ headerLine,
lineList,
fileCounter.getAndIncrement(),
executor,
@@ -356,11 +426,12 @@ public class TsFileTool {
}
} catch (IOException e) {
- e.printStackTrace();
+ LOGGER.error("Error reading file", e);
}
}
private static void submitChunk(
+ String headerLine,
List<String> lineList,
int fileNumber,
ExecutorService executor,
@@ -371,12 +442,28 @@ public class TsFileTool {
() -> {
boolean isSuccess;
if (isSingleFile) {
- isSuccess = writeTsFile(fileName + ".tsfile", lineList);
+ isSuccess = writeTsFile(fileAbsolutePath, fileName + ".tsfile",
lineList);
} else {
- isSuccess = writeTsFile(fileName + "_" + fileNumber + ".tsfile",
lineList);
+ isSuccess =
+ writeTsFile(fileAbsolutePath, fileName + "_" + fileNumber +
".tsfile", lineList);
}
if (!isSuccess) {
- cpFile(fileAbsolutePath, failedDirectoryStr);
+ if (isSingleFile) {
+ cpFile(fileAbsolutePath, failedDirectoryStr);
+ File tsfile = new File(outputDirectoryStr, fileName + ".tsfile");
+ deleteFile(tsfile);
+ } else {
+ writeToNewCSV(
+ headerLine, fileAbsolutePath, lineList, fileName + "_" +
fileNumber + ".csv");
+ File tsfile = new File(outputDirectoryStr, fileName + "_" +
fileNumber + ".tsfile");
+ deleteFile(tsfile);
+ }
+ } else {
+ String tsFileName = fileName + "_" + fileNumber + ".tsfile";
+ if (isSingleFile) {
+ tsFileName = fileName + ".tsfile";
+ }
+ LOGGER.info(tsFileName + " successfully generated");
}
});
}
@@ -402,7 +489,7 @@ public class TsFileTool {
if (cmd.hasOption("h")) {
printHelp(options);
- return;
+ System.exit(0);
}
if (cmd.hasOption("s")) {
@@ -428,7 +515,7 @@ public class TsFileTool {
failedDirectoryStr = "failed";
}
} catch (ParseException e) {
- e.printStackTrace();
+ LOGGER.error("Error parsing command line options", e);
}
}
@@ -469,23 +556,26 @@ public class TsFileTool {
}
private static boolean validateParams() {
- if (inputDirectoryStr == null
- || inputDirectoryStr.isEmpty()
- || outputDirectoryStr == null
- || outputDirectoryStr.isEmpty()
- || schemaPathStr == null
- || schemaPathStr.isEmpty()) {
- LOGGER.error("Missing required parameters. Please provide --source,
--target, and --schema.");
+ if (inputDirectoryStr == null || inputDirectoryStr.isEmpty()) {
+ LOGGER.error("Missing required parameters.--source/-s is a required");
+ return false;
+ }
+ if (outputDirectoryStr == null || outputDirectoryStr.isEmpty()) {
+ LOGGER.error("Missing required parameters. --target/-t is a required");
+ return false;
+ }
+ if (schemaPathStr == null || schemaPathStr.isEmpty()) {
+ LOGGER.error("Missing required parameters. --schema is a required");
return false;
}
File sourceDir = new File(inputDirectoryStr);
if (!sourceDir.exists()) {
- LOGGER.error("Source directory(" + sourceDir + ") does not exist.");
+ LOGGER.error(sourceDir + " directory or file does not exist.");
return false;
}
File schemaFile = new File(schemaPathStr);
if (!schemaFile.exists()) {
- LOGGER.error("Schema file(" + schemaPathStr + ") does not exist.");
+ LOGGER.error(schemaPathStr + " schema file does not exist.");
return false;
}
if (THREAD_COUNT <= 0) {
diff --git
a/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java
b/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java
index 455ab1c5..ddbe32e8 100644
--- a/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java
+++ b/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java
@@ -31,8 +31,10 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
@@ -45,8 +47,12 @@ import static org.junit.Assert.assertTrue;
public class TsfiletoolsTest {
private final String testDir = "target" + File.separator + "csvTest";
private final String csvFile = testDir + File.separator + "data.csv";
+
+ private final String wrongCsvFile = testDir + File.separator +
"dataWrong.csv";
private final String schemaFile = testDir + File.separator +
"schemaFile.txt";
+ private final String failedDir = testDir + File.separator + "failed";
+
float[] tmpResult2 = new float[20];
float[] tmpResult3 = new float[20];
float[] tmpResult5 = new float[20];
@@ -55,6 +61,7 @@ public class TsfiletoolsTest {
public void setUp() {
new File(testDir).mkdirs();
genCsvFile(20);
+ genWrongCsvFile(100);
genSchemaFile();
}
@@ -71,12 +78,18 @@ public class TsfiletoolsTest {
writer.write("null_format=\\N");
writer.newLine();
writer.newLine();
+ writer.write("id_columns");
+ writer.newLine();
+ writer.write("tmp1");
+ writer.newLine();
writer.write("time_column=time");
writer.newLine();
writer.write("csv_columns");
writer.newLine();
writer.write("time INT64,");
writer.newLine();
+ writer.write("tmp1 TEXT,");
+ writer.newLine();
writer.write("tmp2 FLOAT,");
writer.newLine();
writer.write("tmp3 FLOAT,");
@@ -85,20 +98,50 @@ public class TsfiletoolsTest {
writer.newLine();
writer.write("tmp5 FLOAT");
} catch (IOException e) {
- e.printStackTrace();
+ throw new RuntimeException("Failed to generate schema file", e);
+ }
+ }
+
+ public void genWrongCsvFile(int rows) {
+
+ try (BufferedWriter writer = new BufferedWriter(new
FileWriter(wrongCsvFile))) {
+ writer.write("time,tmp1,tmp2,tmp3,tmp4,tmp5");
+ writer.newLine();
+ Random random = new Random();
+ long timestamp = System.currentTimeMillis();
+
+ for (int i = 0; i < rows; i++) {
+ timestamp = timestamp + i;
+ String tmp1 = "s1";
+ float tmp2 = random.nextFloat();
+ float tmp3 = random.nextFloat();
+ float tmp4 = random.nextFloat();
+ float tmp5 = random.nextFloat();
+ if (i % 99 == 0) {
+ writer.write(
+ timestamp + "aa" + "," + tmp1 + "," + tmp2 + "," + tmp3 + "," +
tmp4 + "," + tmp5);
+ } else {
+ writer.write(timestamp + "," + tmp1 + "," + tmp2 + "," + tmp3 + ","
+ tmp4 + "," + tmp5);
+ }
+
+ writer.newLine();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to generate wrong CSV file", e);
}
}
public void genCsvFile(int rows) {
try (BufferedWriter writer = new BufferedWriter(new FileWriter(csvFile))) {
- writer.write("time,tmp2,tmp3,tmp4,tmp5");
+ writer.write("time,tmp1,tmp2,tmp3,tmp4,tmp5");
writer.newLine();
Random random = new Random();
long timestamp = System.currentTimeMillis();
for (int i = 0; i < rows; i++) {
timestamp = timestamp + i;
+ String tmp1 = "s1";
float tmp2 = random.nextFloat();
float tmp3 = random.nextFloat();
float tmp4 = random.nextFloat();
@@ -106,11 +149,11 @@ public class TsfiletoolsTest {
tmpResult2[i] = tmp2;
tmpResult3[i] = tmp3;
tmpResult5[i] = tmp5;
- writer.write(timestamp + "," + tmp2 + "," + tmp3 + "," + tmp4 + "," +
tmp5);
+ writer.write(timestamp + "," + tmp1 + "," + tmp2 + "," + tmp3 + "," +
tmp4 + "," + tmp5);
writer.newLine();
}
} catch (IOException e) {
- e.printStackTrace();
+ throw new RuntimeException("Failed to generate CSV file", e);
}
}
@@ -155,4 +198,28 @@ public class TsfiletoolsTest {
assertEquals(20, cnt);
}
}
+
+ @Test
+ public void testCsvToTsfileFailed() {
+ String scFilePath = new File(schemaFile).getAbsolutePath();
+ String csvFilePath = new File(wrongCsvFile).getAbsolutePath();
+ String targetPath = new File(testDir).getAbsolutePath();
+ String fd = new File(failedDir).getAbsolutePath();
+ String[] args =
+ new String[] {
+ "-s" + csvFilePath, "-schema" + scFilePath, "-t" + targetPath,
"-fail_dir" + fd
+ };
+ TsFileTool.main(args);
+ assertTrue(new File(failedDir + File.separator +
"dataWrong.csv").exists());
+ try (BufferedReader br =
+ new BufferedReader(new FileReader(failedDir + File.separator +
"dataWrong.csv"))) {
+ int num = 0;
+ while (br.readLine() != null) {
+ num++;
+ }
+ assertEquals(101, num);
+ } catch (IOException e) {
+ throw new RuntimeException("IOException occurred while reading file", e);
+ }
+ }
}
