Repository: incubator-carbondata Updated Branches: refs/heads/master 6f221ae1e -> d8d4c98c7
If table block size if specified for its max value 2048, then while converting it into bytes it goes out of integer range and hence a negative value is assigned for blocklet size. Because of this exception is thrown. Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/40de6307 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/40de6307 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/40de6307 Branch: refs/heads/master Commit: 40de6307851f91f976681fcf3ad4e709b58ce245 Parents: 6f221ae Author: manishgupta88 <tomanishgupt...@gmail.com> Authored: Tue Mar 21 14:13:06 2017 +0530 Committer: Venkata Ramana G <ramana.gollam...@huawei.com> Committed: Thu Mar 23 14:37:09 2017 +0530 ---------------------------------------------------------------------- .../dataload/TestTableLevelBlockSize.scala | 27 ++++++++++++++++++++ .../store/writer/AbstractFactDataWriter.java | 4 +-- .../writer/v3/CarbonFactDataWriterImplV3.java | 4 +-- 3 files changed, 31 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/40de6307/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala index 710fd61..db23db5 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala @@ -39,6 +39,7 @@ class TestTableLevelBlockSize extends QueryTest with BeforeAndAfterAll{ sql("DROP TABLE IF EXISTS table_blocksize1") sql("DROP TABLE IF EXISTS table_blocksize2") sql("DROP TABLE IF EXISTS table_blocksize3") + sql("DROP TABLE IF EXISTS table_max_block_size") } test("Value test: set table level blocksize value beyong [1,2048]") { @@ -109,10 +110,36 @@ class TestTableLevelBlockSize extends QueryTest with BeforeAndAfterAll{ } + test("test block size for its max value") { + sql( + """ + CREATE TABLE IF NOT EXISTS table_max_block_size + (ID Int, date Timestamp, country String, + name String, phonetype String, serialname String, salary Int) + STORED BY 'org.apache.carbondata.format' + TBLPROPERTIES('table_blocksize'='2048 MB') + """) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd") + sql(s""" + LOAD DATA LOCAL INPATH '$testData2' into table table_max_block_size + """) + checkAnswer( + sql(""" + SELECT country, count(salary) AS amount + FROM table_max_block_size + WHERE country IN ('china','france') + GROUP BY country + """), + Seq(Row("china", 96), Row("france", 1)) + ) + } + override def afterAll { sql("DROP TABLE IF EXISTS table_blocksize1") sql("DROP TABLE IF EXISTS table_blocksize2") sql("DROP TABLE IF EXISTS table_blocksize3") + sql("DROP TABLE IF EXISTS table_max_block_size") CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/40de6307/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java index d206057..51f8960 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java @@ -113,7 +113,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter< /** * data file size; */ - protected int fileSizeInBytes; + protected long fileSizeInBytes; /** * file count will be used to give sequence number to the data file */ @@ -163,7 +163,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter< CarbonProperties propInstance = CarbonProperties.getInstance(); // if blocksize=2048, then 2048*1024*1024 will beyond the range of Int this.fileSizeInBytes = - dataWriterVo.getTableBlocksize() * CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR + (long) dataWriterVo.getTableBlocksize() * CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR * CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR; this.spaceReservedForBlockMetaSize = Integer.parseInt(propInstance .getProperty(CarbonCommonConstants.CARBON_BLOCK_META_RESERVED_SPACE, http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/40de6307/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java index 108fef4..57e03bd 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java @@ -61,11 +61,11 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]> */ private DataWriterHolder dataWriterHolder; - private int blockletSize; + private long blockletSize; public CarbonFactDataWriterImplV3(CarbonDataWriterVo dataWriterVo) { super(dataWriterVo); - blockletSize = Integer.parseInt(CarbonProperties.getInstance() + blockletSize = Long.parseLong(CarbonProperties.getInstance() .getProperty(CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB, CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE)) * CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR
