[33/50] [abbrv] carbondata git commit: [CARBONDATA-2727][BloomDataMap] Support create bloom datamap on newly added column

2018-07-17 Thread jackylk 
[CARBONDATA-2727][BloomDataMap] Support create bloom datamap on newly added 
column

Add a result collector with rowId infomation for datamap rebuild if table 
schema is changed;
Use keygenerator to retrieve surrogate value of dictIndexColumn from query 
result;

This closes #2490


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/81038f55
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/81038f55
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/81038f55

Branch: refs/heads/carbonstore
Commit: 81038f55ef9a582f82305378988f603ded76e524
Parents: aec47e0
Author: Manhua 
Authored: Wed Jul 11 19:39:31 2018 +0800
Committer: xuchuanyin 
Committed: Tue Jul 17 23:31:43 2018 +0800

--
 .../scan/collector/ResultCollectorFactory.java  |  31 ++---
 ...RowIdRestructureBasedRawResultCollector.java | 138 +++
 .../bloom/AbstractBloomDataMapWriter.java   |  72 +-
 .../bloom/BloomCoarseGrainDataMapFactory.java   |   2 +-
 .../datamap/bloom/BloomDataMapBuilder.java  |   8 ++
 .../datamap/bloom/BloomDataMapWriter.java   |  72 ++
 .../datamap/IndexDataMapRebuildRDD.scala| 131 +++---
 .../bloom/BloomCoarseGrainDataMapSuite.scala|  96 +
 8 files changed, 413 insertions(+), 137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/81038f55/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
index ea4afd1..e0a0b90 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
@@ -18,15 +18,7 @@ package org.apache.carbondata.core.scan.collector;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
-import 
org.apache.carbondata.core.scan.collector.impl.AbstractScannedResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.DictionaryBasedResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.DictionaryBasedVectorResultCollector;
-import org.apache.carbondata.core.scan.collector.impl.RawBasedResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RestructureBasedDictionaryResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RestructureBasedRawResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RestructureBasedVectorResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RowIdBasedResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RowIdRawBasedResultCollector;
+import org.apache.carbondata.core.scan.collector.impl.*;
 import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
 
 /**
@@ -51,14 +43,21 @@ public class ResultCollectorFactory {
 AbstractScannedResultCollector scannerResultAggregator = null;
 if (blockExecutionInfo.isRawRecordDetailQuery()) {
   if (blockExecutionInfo.isRestructuredBlock()) {
-LOGGER.info("Restructure based raw collector is used to scan and 
collect the data");
-scannerResultAggregator = new 
RestructureBasedRawResultCollector(blockExecutionInfo);
-  } else if (blockExecutionInfo.isRequiredRowId()) {
-LOGGER.info("RowId based raw collector is used to scan and collect the 
data");
-scannerResultAggregator = new 
RowIdRawBasedResultCollector(blockExecutionInfo);
+if (blockExecutionInfo.isRequiredRowId()) {
+  LOGGER.info("RowId Restructure based raw ollector is used to scan 
and collect the data");
+  scannerResultAggregator = new 
RowIdRestructureBasedRawResultCollector(blockExecutionInfo);
+} else {
+  LOGGER.info("Restructure based raw collector is used to scan and 
collect the data");
+  scannerResultAggregator = new 
RestructureBasedRawResultCollector(blockExecutionInfo);
+}
   } else {
-LOGGER.info("Row based raw collector is used to scan and collect the 
data");
-scannerResultAggregator = new 
RawBasedResultCollector(blockExecutionInfo);
+if (blockExecutionInfo.isRequiredRowId()) {
+  LOGGER.info("RowId based raw collector is used to scan and collect 
the data");
+  scannerResultAggregator = new 
RowIdRawBasedResultCollector(blockExecutionInfo);
+} else {
+  LOGGER.info("Row based raw collector is used to scan and collect the 
data");
+

carbondata git commit: [CARBONDATA-2727][BloomDataMap] Support create bloom datamap on newly added column

2018-07-17 Thread xuchuanyin 
Repository: carbondata
Updated Branches:
  refs/heads/master aec47e06f -> 81038f55e


[CARBONDATA-2727][BloomDataMap] Support create bloom datamap on newly added 
column

Add a result collector with rowId infomation for datamap rebuild if table 
schema is changed;
Use keygenerator to retrieve surrogate value of dictIndexColumn from query 
result;

This closes #2490


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/81038f55
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/81038f55
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/81038f55

Branch: refs/heads/master
Commit: 81038f55ef9a582f82305378988f603ded76e524
Parents: aec47e0
Author: Manhua 
Authored: Wed Jul 11 19:39:31 2018 +0800
Committer: xuchuanyin 
Committed: Tue Jul 17 23:31:43 2018 +0800

--
 .../scan/collector/ResultCollectorFactory.java  |  31 ++---
 ...RowIdRestructureBasedRawResultCollector.java | 138 +++
 .../bloom/AbstractBloomDataMapWriter.java   |  72 +-
 .../bloom/BloomCoarseGrainDataMapFactory.java   |   2 +-
 .../datamap/bloom/BloomDataMapBuilder.java  |   8 ++
 .../datamap/bloom/BloomDataMapWriter.java   |  72 ++
 .../datamap/IndexDataMapRebuildRDD.scala| 131 +++---
 .../bloom/BloomCoarseGrainDataMapSuite.scala|  96 +
 8 files changed, 413 insertions(+), 137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/81038f55/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
index ea4afd1..e0a0b90 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
@@ -18,15 +18,7 @@ package org.apache.carbondata.core.scan.collector;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
-import 
org.apache.carbondata.core.scan.collector.impl.AbstractScannedResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.DictionaryBasedResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.DictionaryBasedVectorResultCollector;
-import org.apache.carbondata.core.scan.collector.impl.RawBasedResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RestructureBasedDictionaryResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RestructureBasedRawResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RestructureBasedVectorResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RowIdBasedResultCollector;
-import 
org.apache.carbondata.core.scan.collector.impl.RowIdRawBasedResultCollector;
+import org.apache.carbondata.core.scan.collector.impl.*;
 import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
 
 /**
@@ -51,14 +43,21 @@ public class ResultCollectorFactory {
 AbstractScannedResultCollector scannerResultAggregator = null;
 if (blockExecutionInfo.isRawRecordDetailQuery()) {
   if (blockExecutionInfo.isRestructuredBlock()) {
-LOGGER.info("Restructure based raw collector is used to scan and 
collect the data");
-scannerResultAggregator = new 
RestructureBasedRawResultCollector(blockExecutionInfo);
-  } else if (blockExecutionInfo.isRequiredRowId()) {
-LOGGER.info("RowId based raw collector is used to scan and collect the 
data");
-scannerResultAggregator = new 
RowIdRawBasedResultCollector(blockExecutionInfo);
+if (blockExecutionInfo.isRequiredRowId()) {
+  LOGGER.info("RowId Restructure based raw ollector is used to scan 
and collect the data");
+  scannerResultAggregator = new 
RowIdRestructureBasedRawResultCollector(blockExecutionInfo);
+} else {
+  LOGGER.info("Restructure based raw collector is used to scan and 
collect the data");
+  scannerResultAggregator = new 
RestructureBasedRawResultCollector(blockExecutionInfo);
+}
   } else {
-LOGGER.info("Row based raw collector is used to scan and collect the 
data");
-scannerResultAggregator = new 
RawBasedResultCollector(blockExecutionInfo);
+if (blockExecutionInfo.isRequiredRowId()) {
+  LOGGER.info("RowId based raw collector is used to scan and collect 
the data");
+  scannerResultAggregator = new 
RowIdRawBasedResultCollector(blockExecutionInfo);
+} else {
+  LOGGER.info("Row based raw