[CARBONDATA-2614] Fix the error when using FG in search mode and the prune result is none
the prune result is none, and can not set datamapWritePath, which will not generate bitSegGroup in org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl#applyFilter(org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks, boolean), it mean the bitSetGroup is null. It will throw Fix the error when using FG in search mode and the prune result is none This closes #2378 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5593d164 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5593d164 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5593d164 Branch: refs/heads/carbonstore Commit: 5593d1646b075847a84ac2891fd25e78201451fe Parents: 60dfdd3 Author: xubo245 <xub...@huawei.com> Authored: Fri Jun 15 18:50:26 2018 +0800 Committer: Jacky Li <jacky.li...@qq.com> Committed: Mon Jun 18 20:29:59 2018 +0800 ---------------------------------------------------------------------- .../apache/carbondata/core/scan/model/QueryModel.java | 13 +++++++++++++ .../apache/carbondata/hadoop/CarbonRecordReader.java | 2 +- .../testsuite/detailquery/SearchModeTestCase.scala | 1 + .../carbondata/store/worker/SearchRequestHandler.java | 1 + 4 files changed, 16 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java index de11d11..55dafb9 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java @@ -109,6 +109,11 @@ public class QueryModel { */ private boolean requiredRowId; + /** + * whether it is FG with search mode + */ + private boolean isFG; + private QueryModel(CarbonTable carbonTable) { tableBlockInfos = new ArrayList<TableBlockInfo>(); invalidSegmentIds = new ArrayList<>(); @@ -370,6 +375,14 @@ public class QueryModel { this.requiredRowId = requiredRowId; } + public boolean isFG() { + return isFG; + } + + public void setFG(boolean FG) { + isFG = FG; + } + @Override public String toString() { return String.format("scan on table %s.%s, %d projection columns with filter (%s)", http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java index 4911e41..6b56382 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java @@ -80,7 +80,7 @@ public class CarbonRecordReader<T> extends AbstractRecordReader<T> { } // It should use the exists tableBlockInfos if tableBlockInfos of queryModel is not empty // otherwise the prune is no use before this method - if (queryModel.getTableBlockInfos().isEmpty()) { + if (!queryModel.isFG()) { List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList); queryModel.setTableBlockInfos(tableBlockInfoList); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala index 3e6adaf..001f6c0 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala @@ -111,6 +111,7 @@ class SearchModeTestCase extends QueryTest with BeforeAndAfterAll { } test("test lucene datamap with search mode") { + sql("set carbon.search.enabled = true") sql("DROP DATAMAP IF EXISTS dm ON TABLE main") sql("CREATE DATAMAP dm ON TABLE main USING 'lucene' DMProperties('INDEX_COLUMNS'='id') ") checkAnswer(sql("SELECT * FROM main WHERE TEXT_MATCH('id:100000')"), http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java ---------------------------------------------------------------------- diff --git a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java index 4bfadce..0a3110e 100644 --- a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java +++ b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java @@ -207,6 +207,7 @@ public class SearchRequestHandler { LOG.info(String.format("[SearchId:%d] pruned using FG DataMap, pruned blocks: %d", queryId, blockToRead.size())); queryModel.setTableBlockInfos(blockToRead); + queryModel.setFG(true); return queryModel; }