Repository: carbondata Updated Branches: refs/heads/branch-1.4 c8da20d5c -> 1c2857b79
[CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure case for creation of multi reader for non-transactional table Issue : CarbonIterator inside CarbonRecordReader was keeping reference of RowBatch and it is not being closed inside CarbonRecordReader. sort_column with measure was considering all the dimension column along with given column. if creation of one CarbonReader for non transactional table is failed then we are not able to create another CarbonReader. Solution : close() called inside hasNext() to clear the previous iterator before iterating over the next CarbonReader.. if sortcolumn is not empty and sortcolumnsList contains the fields check is added to finally in sortcolumn props. Clear the datamap in catch block, if creation of CarbonReader is failed. This closes #2362 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1c2857b7 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1c2857b7 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1c2857b7 Branch: refs/heads/branch-1.4 Commit: 1c2857b79939ed96e53868f02a1d961cfecb7814 Parents: c8da20d Author: rahul <rahul.ku...@knoldus.in> Authored: Tue Jun 5 13:09:36 2018 +0530 Committer: kumarvishal09 <kumarvishal1...@gmail.com> Committed: Tue Jun 12 17:05:31 2018 +0530 ---------------------------------------------------------------------- .../createTable/TestNonTransactionalCarbonTable.scala | 7 ++++--- .../java/org/apache/carbondata/sdk/file/CarbonReader.java | 2 ++ .../org/apache/carbondata/sdk/file/CarbonWriterBuilder.java | 5 ----- 3 files changed, 6 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c2857b7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala index b275bb8..805fc71 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala @@ -368,8 +368,9 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { |'carbondata' LOCATION |'$writerPath' """.stripMargin) - checkExistence(sql("describe formatted sdkOutputTable"), true, "age") - + checkExistence(sql("describe formatted sdkOutputTable"), true, "SORT_COLUMNS age") + checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS name,age") + checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS age,name") buildTestDataSingleFile() assert(new File(writerPath).exists()) sql("DROP TABLE IF EXISTS sdkOutputTable") @@ -402,7 +403,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { intercept[RuntimeException] { buildTestDataWithSortColumns(List("")) } - + assert(!(new File(writerPath).exists())) cleanTestData() } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c2857b7/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java index 9af710f..be809e6 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java @@ -74,6 +74,8 @@ public class CarbonReader<T> { return false; } else { index++; + // current reader is closed + currentReader.close(); currentReader = readers.get(index); return currentReader.nextKeyValue(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c2857b7/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java index bd64568..0f670fe 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java @@ -484,11 +484,6 @@ public class CarbonWriterBuilder { if (isSortColumn > -1) { columnSchema.setSortColumn(true); sortColumnsSchemaList[isSortColumn] = columnSchema; - } else if (!sortColumnsList.isEmpty() && columnSchema.isDimensionColumn() - && columnSchema.getNumberOfChild() < 1) { - columnSchema.setSortColumn(true); - sortColumnsSchemaList[i] = columnSchema; - i++; } } }