Genius-pig commented on a change in pull request #1914:
URL: https://github.com/apache/iotdb/pull/1914#discussion_r526936444
##########
File path: cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
##########
@@ -119,296 +89,90 @@ private static Options createOptions() {
/**
* Data from csv To tsfile.
*/
- private static void loadDataFromCSV(File file, int index) {
- statement = null;
+ private static void loadDataFromCSV(File file) {
int fileLine;
try {
fileLine = getFileLineCount(file);
} catch (IOException e) {
System.out.println("Failed to import file: " + file.getName());
return;
}
- File errorFile = new File(errorInsertInfo + index);
- if (!errorFile.exists()) {
- try {
- errorFile.createNewFile();
- } catch (IOException e) {
- System.out.println("Cannot create a errorFile because: " +
e.getMessage());
- return;
- }
- }
System.out.println("Start to import data from: " + file.getName());
- errorFlag = true;
- try(BufferedReader br = new BufferedReader(new FileReader(file));
- BufferedWriter bw = new BufferedWriter(new FileWriter(errorFile));
+ try (BufferedReader br = new BufferedReader(new FileReader(file));
ProgressBar pb = new ProgressBar("Import from: " + file.getName(),
fileLine)) {
pb.setExtraMessage("Importing...");
String header = br.readLine();
-
- bw.write("From " + file.getAbsolutePath());
- bw.newLine();
- bw.newLine();
- bw.write(header);
- bw.newLine();
- bw.newLine();
-
- // storage csv table head info
- Map<String, ArrayList<Integer>> deviceToColumn = new HashMap<>();
- // storage csv table head info
- List<String> colInfo = new ArrayList<>();
- // storage csv device sensor info, corresponding csv table head
- List<String> headInfo = new ArrayList<>();
-
- String[] strHeadInfo = header.split(",");
- if (strHeadInfo.length <= 1) {
- System.out.println("The CSV file "+ file.getName() +" illegal, please
check first line");
+ String[] cols = splitCsvLine(header);
+ if (cols.length <= 1) {
+ System.out.println("The CSV file " + file.getName() + " illegal,
please check first line");
return;
}
- long startTime = System.currentTimeMillis();
- Map<String, String> timeseriesDataType = new HashMap<>();
+ List<String> devices = new ArrayList<>();
+ List<Long> times = new ArrayList<>();
+ List<List<String>> measurementsList = new ArrayList<>();
+ List<List<String>> valuesList = new ArrayList<>();
+ Map<String, List<Integer>> devicesToPositions = new HashMap<>();
+ Map<String, List<String>> devicesToMeasurements = new HashMap<>();
- boolean success = queryDatabaseMeta(strHeadInfo, file, bw,
timeseriesDataType, headInfo,
- deviceToColumn, colInfo);
- if (!success) {
- errorFlag = false;
- return;
+ for (int i = 1; i < cols.length; i++) {
+ splitColToDeviceAndMeasurement(cols[i], devicesToPositions,
devicesToMeasurements, i);
}
- statement = connection.createStatement();
-
+ int lineNumber = 0;
+ String line;
+ while ((line = br.readLine()) != null) {
+ cols = splitCsvLine(line);
+ lineNumber++;
+ for (Entry<String, List<Integer>> deviceToPositions :
devicesToPositions
+ .entrySet()) {
+ String device = deviceToPositions.getKey();
+ devices.add(device);
+
+ times.add(Long.parseLong(cols[0]));
+
+ List<String> values = new ArrayList<>();
+ for (int position : deviceToPositions.getValue()) {
+ if (cols[position].equals("") && cols[position].equals("null")) {
Review comment:
remove, "" represent "";
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
