ShreelekhyaG commented on a change in pull request #4228:
URL: https://github.com/apache/carbondata/pull/4228#discussion_r731565065
##########
File path:
processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
##########
@@ -237,21 +237,31 @@ public boolean getIsColumnDictionary() {
@Override
public void writeByteArray(Object input, DataOutputStream dataOutputStream,
- BadRecordLogHolder logHolder, Boolean isWithoutConverter) throws
IOException {
+ BadRecordLogHolder logHolder, Boolean isWithoutConverter, boolean
isEmptyBadRecord)
+ throws IOException {
String parsedValue = null;
// write null value
if (null == input || ((this.carbonDimension.getDataType() ==
DataTypes.STRING
|| this.carbonDimension.getDataType() == DataTypes.VARCHAR) &&
input.equals(nullFormat))) {
updateNullValue(dataOutputStream, logHolder);
return;
}
+ if (input.equals("") && isEmptyBadRecord) {
+ CarbonBadRecordUtil.setErrorMessage(logHolder,
carbonDimension.getColName(),
Review comment:
Done. Used new method updateEmptyValue
##########
File path:
integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordEmptyDataTest.scala
##########
@@ -101,6 +101,60 @@ class BadRecordEmptyDataTest extends QueryTest with
BeforeAndAfterAll {
}
}
+ test("Test complex type with empty values and IS_EMPTY_DATA_BAD_RECORD
property") {
+ sql("DROP TABLE IF EXISTS complexcarbontable")
+ sql("DROP TABLE IF EXISTS complexhivetable")
+ sql(
+ "create table complexcarbontable(deviceInformationId int, channelsId
string, ROMSize " +
+ "string, ROMName String, purchasedate string, file
struct<school:array<string>, age:int>," +
+ " MAC map<string, int>, locationinfo array<struct<ActiveAreaId:int,
ActiveCountry:string, " +
+ "ActiveProvince:string, Activecity:string, ActiveDistrict:string,
ActiveStreet:string>>, " +
+ "proddate
struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
+ "double,contractNumber double, st struct<school:struct<a:string,b:int>,
age:int>," +
+ "ar array<array<string>>) STORED AS carbondata")
+ val exception = intercept[Exception] (sql("LOAD DATA local inpath '" +
resourcesPath +
+ "/complextypeWithEmptyRecords.csv' INTO table complexcarbontable
OPTIONS('DELIMITER'=','," +
+ "'QUOTECHAR'='\"',
'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName," +
+
"purchasedate,file,MAC,locationinfo,proddate,gamePointId,contractNumber,st,ar',
" +
+ "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':', " +
+ "'bad_records_logger_enable'='true','IS_EMPTY_DATA_BAD_RECORD'='true'
," +
+ "'bad_records_action'='fail')"))
+ assert(exception.getMessage.contains(
+ "The value with column name file.age and column data type INT is not a
valid INT type."))
+ sql("LOAD DATA local inpath '" + resourcesPath +
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
