Indhumathi27 commented on a change in pull request #4228:
URL: https://github.com/apache/carbondata/pull/4228#discussion_r723094923
##########
File path:
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordEmptyDataTest.scala
##########
@@ -101,6 +101,47 @@ class BadRecordEmptyDataTest extends QueryTest with
BeforeAndAfterAll {
}
}
+ test("Test complex type with empty values and IS_EMPTY_DATA_BAD_RECORD
property") {
+ sql("DROP TABLE IF EXISTS complexcarbontable")
+ sql("DROP TABLE IF EXISTS complexhivetable")
+ sql(
+ "create table complexcarbontable(deviceInformationId int, channelsId
string, ROMSize " +
+ "string, ROMName String, purchasedate string, mobile struct<imei:string,
imsi:int>, MAC " +
+ "array<string>, locationinfo array<struct<ActiveAreaId:int,
ActiveCountry:string, " +
+ "ActiveProvince:string, Activecity:string, ActiveDistrict:string,
ActiveStreet:string>>, " +
+ "proddate
struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
+ "double,contractNumber double) STORED AS carbondata")
+ sql("LOAD DATA local inpath '" + resourcesPath +
+ "/complextypeWithEmptyRecords.csv' INTO table complexcarbontable
OPTIONS('DELIMITER'=','," +
+ "'QUOTECHAR'='\"',
'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName," +
+
"purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber', " +
+ "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':', " +
+ "'bad_records_logger_enable'='true','IS_EMPTY_DATA_BAD_RECORD'='true' ,"
+
+ "'bad_records_action'='ignore')")
Review comment:
can you add one validation with 'IS_EMPTY_DATA_BAD_RECORD'='true' and
'bad_records_action'='fail'
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
