[CARBONDATA-2455]Fix _System Folder creation and lucene AND,OR,NOT Filter fix
This closes #2281 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/61afa42d Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/61afa42d Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/61afa42d Branch: refs/heads/spark-2.3 Commit: 61afa42daafa03a7e8e0eb980cb6d667d6304785 Parents: 6b70b7e Author: Indhumathi27 <indhumathi...@gmail.com> Authored: Tue May 8 15:36:47 2018 +0530 Committer: Jacky Li <jacky.li...@qq.com> Committed: Fri May 11 09:29:19 2018 +0800 ---------------------------------------------------------------------- .../schema/table/DiskBasedDMSchemaStorageProvider.java | 2 +- .../carbondata/datamap/lucene/LuceneDataMapWriter.java | 1 - .../datamap/lucene/LuceneFineGrainDataMap.java | 3 +-- .../cluster/sdv/generated/LuceneTestCase.scala | 7 +++---- .../datamap/lucene/LuceneFineGrainDataMapSuite.scala | 11 +++++------ 5 files changed, 10 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java index e1a929c..4ebbcd0 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java @@ -55,7 +55,7 @@ public class DiskBasedDMSchemaStorageProvider implements DataMapSchemaStoragePro private Set<DataMapSchema> dataMapSchemas = new HashSet<>(); public DiskBasedDMSchemaStorageProvider(String storePath) { - this.storePath = storePath; + this.storePath = CarbonUtil.checkAndAppendHDFSUrl(storePath); this.mdtFilePath = storePath + CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile"; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java ---------------------------------------------------------------------- diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java index 12fa1ef..3615936 100644 --- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java +++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java @@ -274,7 +274,6 @@ public class LuceneDataMapWriter extends DataMapWriter { } } else if (type == DataTypes.STRING) { byte[] value = (byte[]) data; - // TODO: how to get string value String strValue = null; try { strValue = new String(value, 2, value.length - 2, "UTF-8"); http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java ---------------------------------------------------------------------- diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java index 8e457b7..f8d1b12 100644 --- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java +++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java @@ -205,8 +205,7 @@ public class LuceneFineGrainDataMap extends FineGrainDataMap { queryParser.setAllowLeadingWildcard(true); Query query; try { - // always send lowercase string to lucene as it is case sensitive - query = queryParser.parse(strQuery.toLowerCase()); + query = queryParser.parse(strQuery); } catch (ParseException e) { String errorMessage = String.format( "failed to filter block with query %s, detail is %s", strQuery, e.getMessage()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala index d111594..54d0d11 100644 --- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala @@ -110,14 +110,13 @@ class LuceneTestCase extends QueryTest with BeforeAndAfterAll { checkAnswer(sql("SELECT * FROM datamap_main WHERE TEXT_MATCH('country:ch*')"), sql("select * from datamap_main where country like 'ch%'")) checkAnswer(sql( - "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:ch*') AND TEXT_MATCH('name:aa*')"), + "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:ch* AND name:aa*')"), sql("select * from datamap_main where country like 'ch%' and name like 'aa%'")) checkAnswer(sql( - "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u* or name:aa*')"), + "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u* OR name:aa*')"), sql("select * from datamap_main where country like 'u%'or name like 'aa%'")) checkAnswer(sql( - "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u*') OR TEXT_MATCH('name:aaa1*') AND " + - "TEXT_MATCH('name:aaa2*')"), + "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u* OR (name:aaa1* AND name:aaa2*)')"), sql( "select * from datamap_main where country like 'u%' OR name like 'aaa1%' AND name like " + "'aaa2%'")) http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala index 9981ff5..6d2eb3f 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala @@ -290,7 +290,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { """.stripMargin) sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_table OPTIONS('header'='false')") checkAnswer(sql( - "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n0*') AND TEXT_MATCH(' city:c0*')"), + "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n0* AND city:c0*')"), sql("select * from datamap_test_table where name like 'n0%' and city like 'c0%'")) sql("drop datamap if exists dm on table datamap_test_table") } @@ -311,7 +311,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { """.stripMargin) sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_table OPTIONS('header'='false')") checkAnswer(sql( - "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1*') or TEXT_MATCH('city:c01*')"), + "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1* OR city:c01*')"), sql("select * from datamap_test_table where name like 'n1%' or city like 'c01%'")) sql("drop datamap if exists dm on table datamap_test_table") } @@ -332,8 +332,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { """.stripMargin) sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_table OPTIONS('header'='false')") checkAnswer(sql( - "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1*') OR TEXT_MATCH ('city:c01*') " + - "AND TEXT_MATCH('city:C02*')"), + "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1* OR (city:c01* AND city:c02*)')"), sql( "select * from datamap_test_table where name like 'n1%' OR city like 'c01%' and city like" + " 'c02%'")) @@ -488,7 +487,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { sql("select *from datamap_test_table where name like'n1%' AND not name like 'n2%'")) //check NOT filter with TEXTMATCH wildcard-search using AND on different columns checkAnswer(sql( - "select *from datamap_test_table where TEXT_MATCH('name:n1*')AND TEXT_MATCH('city:c01* NOT " + + "select *from datamap_test_table where TEXT_MATCH('name:n1* AND city:c01* NOT " + "c02*')"), sql("select *from datamap_test_table where name like'n1%' AND not city='c02%'")) sql("drop datamap if exists dm on table datamap_test_table") @@ -716,7 +715,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { sql(s"SELECT * FROM datamap_test5 WHERE city='c020'")) sql("DROP TABLE IF EXISTS datamap_test5") } - + test("test text_match on normal table") { sql("DROP TABLE IF EXISTS table1") sql(