Indhumathi27 commented on a change in pull request #4046:
URL: https://github.com/apache/carbondata/pull/4046#discussion_r542237261
##########
File path:
sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
##########
@@ -1741,6 +1742,246 @@ public boolean accept(File dir, String name) {
}
}
+ @Test
+ public void testReadDateAndTimestampColumnInArray() {
+ String path = "./testWriteFiles";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ Field[] fields = new Field[11];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("shortField", DataTypes.SHORT);
+ fields[2] = new Field("dateField", DataTypes.DATE);
+ fields[3] = new Field("timeField", DataTypes.TIMESTAMP);
+ fields[4] = new Field("varcharField", DataTypes.VARCHAR);
+ fields[5] = new Field("arrayFieldDate",
DataTypes.createArrayType(DataTypes.DATE));
+ fields[6] = new Field("arrayFieldTimestamp",
DataTypes.createArrayType(DataTypes.TIMESTAMP));
+ Map<String, String> map = new HashMap<>();
+ map.put("complex_delimiter_level_1", "#");
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withLoadOptions(map)
+ .withCsvInput(new Schema(fields))
+ .writtenBy("CarbonReaderTest")
+ .build();
+
+ for (int i = 0; i < 10; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i % 10000),
+ "2019-03-02",
+ "2019-02-12 03:03:34",
+ "varchar",
+ "2019-03-02#2019-03-03#2019-03-04#2019-03-05",
+ "2019-02-12 03:03:34#2019-02-12 03:03:38#2019-02-12
03:03:41#2019-02-12 03:12:34"
+ };
+ writer.write(row2);
+ }
+ writer.close();
+ File[] dataFiles = new File(path).listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name == null) {
+ return false;
+ }
+ return name.endsWith("carbondata");
+ }
+ });
+ if (dataFiles == null || dataFiles.length < 1) {
+ throw new RuntimeException("Carbon data file not exists.");
+ }
+ Schema schema = CarbonSchemaReader
+ .readSchema(dataFiles[0].getAbsolutePath())
+ .asOriginOrder();
+ // Transform the schema
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
+ }
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path)
+ .projection(strings)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ assert (row[0].equals("robot" + i));
+ assert (row[2].equals("2019-03-02"));
+ assert (row[3].equals("2019-02-12 03:03:34"));
+ Object[] arrDate = (Object[]) row[5];
+ assert (arrDate[0].equals("2019-03-02"));
+ assert (arrDate[1].equals("2019-03-03"));
+ assert (arrDate[2].equals("2019-03-04"));
+ assert (arrDate[3].equals("2019-03-05"));
+ Object[] arrTimestamp = (Object[]) row[6];
+ assert (arrTimestamp[0].equals("2019-02-12 03:03:34"));
+ assert (arrTimestamp[1].equals("2019-02-12 03:03:38"));
+ assert (arrTimestamp[2].equals("2019-02-12 03:03:41"));
+ assert (arrTimestamp[3].equals("2019-02-12 03:12:34"));
+ i++;
+ }
+ Assert.assertEquals(i, 10);
+ reader.close();
+ FileUtils.deleteDirectory(new File(path));
+ } catch (Throwable e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test public void testReadDateAndTimestampColumnInStruct()
Review comment:
can you add a testcase with Map type having Date/Timestamp as key/value
##########
File path:
sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
##########
@@ -1741,6 +1742,246 @@ public boolean accept(File dir, String name) {
}
}
+ @Test
+ public void testReadDateAndTimestampColumnInArray() {
+ String path = "./testWriteFiles";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ Field[] fields = new Field[11];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("shortField", DataTypes.SHORT);
+ fields[2] = new Field("dateField", DataTypes.DATE);
+ fields[3] = new Field("timeField", DataTypes.TIMESTAMP);
+ fields[4] = new Field("varcharField", DataTypes.VARCHAR);
+ fields[5] = new Field("arrayFieldDate",
DataTypes.createArrayType(DataTypes.DATE));
+ fields[6] = new Field("arrayFieldTimestamp",
DataTypes.createArrayType(DataTypes.TIMESTAMP));
+ Map<String, String> map = new HashMap<>();
+ map.put("complex_delimiter_level_1", "#");
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withLoadOptions(map)
+ .withCsvInput(new Schema(fields))
+ .writtenBy("CarbonReaderTest")
+ .build();
+
+ for (int i = 0; i < 10; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i % 10000),
+ "2019-03-02",
+ "2019-02-12 03:03:34",
+ "varchar",
+ "2019-03-02#2019-03-03#2019-03-04#2019-03-05",
+ "2019-02-12 03:03:34#2019-02-12 03:03:38#2019-02-12
03:03:41#2019-02-12 03:12:34"
+ };
+ writer.write(row2);
+ }
+ writer.close();
+ File[] dataFiles = new File(path).listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name == null) {
+ return false;
+ }
+ return name.endsWith("carbondata");
+ }
+ });
+ if (dataFiles == null || dataFiles.length < 1) {
+ throw new RuntimeException("Carbon data file not exists.");
+ }
+ Schema schema = CarbonSchemaReader
+ .readSchema(dataFiles[0].getAbsolutePath())
+ .asOriginOrder();
+ // Transform the schema
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
+ }
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path)
+ .projection(strings)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ assert (row[0].equals("robot" + i));
+ assert (row[2].equals("2019-03-02"));
+ assert (row[3].equals("2019-02-12 03:03:34"));
+ Object[] arrDate = (Object[]) row[5];
+ assert (arrDate[0].equals("2019-03-02"));
+ assert (arrDate[1].equals("2019-03-03"));
+ assert (arrDate[2].equals("2019-03-04"));
+ assert (arrDate[3].equals("2019-03-05"));
+ Object[] arrTimestamp = (Object[]) row[6];
+ assert (arrTimestamp[0].equals("2019-02-12 03:03:34"));
+ assert (arrTimestamp[1].equals("2019-02-12 03:03:38"));
+ assert (arrTimestamp[2].equals("2019-02-12 03:03:41"));
+ assert (arrTimestamp[3].equals("2019-02-12 03:12:34"));
+ i++;
+ }
+ Assert.assertEquals(i, 10);
+ reader.close();
+ FileUtils.deleteDirectory(new File(path));
+ } catch (Throwable e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test public void testReadDateAndTimestampColumnInStruct()
+ throws IOException, InvalidLoadOptionException, InterruptedException {
Review comment:
Remove Exception if not used
##########
File path:
sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
##########
@@ -1741,6 +1742,246 @@ public boolean accept(File dir, String name) {
}
}
+ @Test
+ public void testReadDateAndTimestampColumnInArray() {
+ String path = "./testWriteFiles";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ Field[] fields = new Field[11];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("shortField", DataTypes.SHORT);
+ fields[2] = new Field("dateField", DataTypes.DATE);
+ fields[3] = new Field("timeField", DataTypes.TIMESTAMP);
+ fields[4] = new Field("varcharField", DataTypes.VARCHAR);
+ fields[5] = new Field("arrayFieldDate",
DataTypes.createArrayType(DataTypes.DATE));
+ fields[6] = new Field("arrayFieldTimestamp",
DataTypes.createArrayType(DataTypes.TIMESTAMP));
+ Map<String, String> map = new HashMap<>();
+ map.put("complex_delimiter_level_1", "#");
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withLoadOptions(map)
+ .withCsvInput(new Schema(fields))
+ .writtenBy("CarbonReaderTest")
+ .build();
+
+ for (int i = 0; i < 10; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i % 10000),
+ "2019-03-02",
+ "2019-02-12 03:03:34",
+ "varchar",
+ "2019-03-02#2019-03-03#2019-03-04#2019-03-05",
+ "2019-02-12 03:03:34#2019-02-12 03:03:38#2019-02-12
03:03:41#2019-02-12 03:12:34"
+ };
+ writer.write(row2);
+ }
+ writer.close();
+ File[] dataFiles = new File(path).listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name == null) {
+ return false;
+ }
+ return name.endsWith("carbondata");
+ }
+ });
+ if (dataFiles == null || dataFiles.length < 1) {
+ throw new RuntimeException("Carbon data file not exists.");
+ }
+ Schema schema = CarbonSchemaReader
+ .readSchema(dataFiles[0].getAbsolutePath())
+ .asOriginOrder();
+ // Transform the schema
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
+ }
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path)
+ .projection(strings)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ assert (row[0].equals("robot" + i));
+ assert (row[2].equals("2019-03-02"));
+ assert (row[3].equals("2019-02-12 03:03:34"));
+ Object[] arrDate = (Object[]) row[5];
+ assert (arrDate[0].equals("2019-03-02"));
+ assert (arrDate[1].equals("2019-03-03"));
+ assert (arrDate[2].equals("2019-03-04"));
+ assert (arrDate[3].equals("2019-03-05"));
+ Object[] arrTimestamp = (Object[]) row[6];
+ assert (arrTimestamp[0].equals("2019-02-12 03:03:34"));
+ assert (arrTimestamp[1].equals("2019-02-12 03:03:38"));
+ assert (arrTimestamp[2].equals("2019-02-12 03:03:41"));
+ assert (arrTimestamp[3].equals("2019-02-12 03:12:34"));
+ i++;
+ }
+ Assert.assertEquals(i, 10);
+ reader.close();
+ FileUtils.deleteDirectory(new File(path));
+ } catch (Throwable e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test public void testReadDateAndTimestampColumnInStruct()
+ throws IOException, InvalidLoadOptionException, InterruptedException {
+ String path = "./testWriteFiles";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ Field[] fields = new Field[3];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+ ArrayList<StructField> structFields = new ArrayList<>();
+ structFields.add(new StructField("dateField", DataTypes.DATE));
+ structFields.add(new StructField("timestampField", DataTypes.TIMESTAMP));
+ fields[2] = new Field("structField",
DataTypes.createStructType(structFields));
+ Map<String, String> map = new HashMap<>();
+ map.put("complex_delimiter_level_1", "#");
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withLoadOptions(map)
+ .withCsvInput(new Schema(fields))
+ .writtenBy("CarbonReaderTest")
+ .build();
+
+ for (int i = 0; i < 10; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i % 10000),
+ "2019-03-02#2019-02-12 03:12:34"
+ };
+ writer.write(row2);
+ }
+ writer.close();
+ File[] dataFiles = new File(path).listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name == null) {
+ return false;
+ }
+ return name.endsWith("carbondata");
+ }
+ });
+ if (dataFiles == null || dataFiles.length < 1) {
+ throw new RuntimeException("Carbon data file not exists.");
+ }
+ Schema schema = CarbonSchemaReader
+ .readSchema(dataFiles[0].getAbsolutePath())
+ .asOriginOrder();
+ // Transform the schema
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
+ }
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path)
+ .projection(strings)
+ .build();
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ assert (row[0].equals("robot" + i));
+ Object[] arr = (Object[]) row[2];
+ assert (arr[0].equals("2019-03-02"));
+ assert (arr[1].equals("2019-02-12 03:12:34"));
+ i++;
+ }
+ Assert.assertEquals(i, 10);
+ reader.close();
+ FileUtils.deleteDirectory(new File(path));
+ } catch (Throwable e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testReadingDateAndTimestampColumnInArrayOfStruct() throws
IOException {
+ String path = "./testWriteFilesArrayStruct";
+ FileUtils.deleteDirectory(new File(path));
+ Field[] fields = new Field[4];
+ fields[0] = new Field("id", DataTypes.STRING);
+ fields[1] = new Field("source", DataTypes.STRING);
+ fields[2] = new Field("usage", DataTypes.STRING);
+ List<StructField> structFieldsList = new ArrayList<>();
+ structFieldsList.add(new StructField("name", DataTypes.STRING));
+ structFieldsList.add(new StructField("type", DataTypes.STRING));
+ structFieldsList.add(new StructField("creation-date", DataTypes.DATE));
+ structFieldsList.add(new StructField("creation-timestamp",
DataTypes.TIMESTAMP));
+ StructField structTypeByList =
+ new StructField("annotation",
DataTypes.createStructType(structFieldsList), structFieldsList);
+ List<StructField> list = new ArrayList<>();
+ list.add(structTypeByList);
+ Field arrayType = new Field("annotations", "array", list);
+ fields[3] = arrayType;
+ try {
+ CarbonWriterBuilder builder = CarbonWriter.builder().outputPath(path);
+ CarbonWriter writer = builder.withCsvInput(new Schema(fields))
+ .writtenBy("complexTest")
+ .build();
+ for (int i = 0; i < 15; i++) {
+ String[] row = new String[]{
+ "robot" + i,
+ String.valueOf(i),
+ i + "." + i,
+ "sunflowers" + (i % 10) + "\002" +
"modelarts/image_classification" + "\002" + "2019-03-30" + "\002" + "2019-03-30
17:22:31"
+ + "\001" +
+ "roses" + (i % 10) + "\002" + "modelarts/image_classification"
+ "\002" + "2019-03-30" + "\002" + "2019-03-30 17:22:31"};
+ writer.write(row);
+ }
+ writer.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ Schema schema = CarbonSchemaReader
+ .readSchema(path)
+ .asOriginOrder();
+ assert (4 == schema.getFieldsLength());
+ Field[] fields1 = schema.getFields();
Review comment:
Remove variable if not used
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
