[12/15] kylin git commit: add back calcite linq4j

2017-01-22 Thread liyang 
add back calcite linq4j


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68e87f08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68e87f08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68e87f08

Branch: refs/heads/master-cdh5.7
Commit: 68e87f08255caf33dd7946f43b3393372a73c68b
Parents: b56bc62
Author: Hongbin Ma 
Authored: Mon Jan 23 11:54:54 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 11:54:54 2017 +0800

--
 pom.xml | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/68e87f08/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 57b7752..e590fd7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -450,6 +450,11 @@
 
 
 
+org.apache.calcite
+calcite-linq4j
+${calcite.version}
+
+
 org.apache.calcite.avatica
 avatica
 ${avatica.version}

[04/15] kylin git commit: KYLIN-2394 merge CALCITE-1592

2017-01-22 Thread liyang 
KYLIN-2394 merge CALCITE-1592


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34f2d3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34f2d3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34f2d3e

Branch: refs/heads/master-cdh5.7
Commit: d34f2d3e2bf691c55077a9a76a89217ae915bbdd
Parents: dfb5fac
Author: Billy Liu 
Authored: Fri Jan 20 15:42:37 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 15:42:37 2017 +0800

--
 .../calcite/sql2rel/SqlToRelConverter.java  | 51 +---
 1 file changed, 23 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d34f2d3e/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 171e299..cc9c9a1 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -935,19 +935,21 @@ public class SqlToRelConverter {
 /**
  * Push down all the NOT logical operators into any IN/NOT IN operators.
  *
+ * @param scope Scope where {@code sqlNode} occurs
  * @param sqlNode the root node from which to look for NOT operators
  * @return the transformed SqlNode representation with NOT pushed down.
  */
-private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+private static SqlNode pushDownNotForIn(SqlValidatorScope scope,
+SqlNode sqlNode) {
 if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
 SqlCall sqlCall = (SqlCall) sqlNode;
 if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
 || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
 SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
 for (int i = 0; i < sqlOperands.length; i++) {
-sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+sqlOperands[i] = pushDownNotForIn(scope, sqlOperands[i]);
 }
-return sqlNode;
+return reg(scope, sqlNode);
 } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
 SqlNode childNode = sqlCall.operand(0);
 assert childNode instanceof SqlCall;
@@ -956,48 +958,34 @@ public class SqlToRelConverter {
 SqlNode[] andOperands = childSqlCall.getOperands();
 SqlNode[] orOperands = new SqlNode[andOperands.length];
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-andOperands[i]);
+orOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]));
 }
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] = pushDownNotForIn(orOperands[i]);
+orOperands[i] = pushDownNotForIn(scope, orOperands[i]);
 }
-return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
-orOperands[0], orOperands[1]);
+return reg(scope, 
SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], 
orOperands[1]));
 } else if (childSqlCall.getOperator() == 
SqlStdOperatorTable.OR) {
 SqlNode[] orOperands = childSqlCall.getOperands();
 SqlNode[] andOperands = new SqlNode[orOperands.length];
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-orOperands[i]);
+andOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]));
 }
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] = pushDownNotForIn(andOperands[i]);
+andOperands[i] = pushDownNotForIn(scope, 
andOperands[i]);
 }
-return 
SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
-

[06/15] kylin git commit: Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs"

2017-01-22 Thread liyang 
Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs"

This reverts commit 7de8aa1203a72bad105ed692f7100535939b03af.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7576a09f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7576a09f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7576a09f

Branch: refs/heads/master-cdh5.7
Commit: 7576a09f29b75f2eb80a1b5ca9c65fe5c81b1144
Parents: 7de8aa1
Author: shaofengshi 
Authored: Sun Jan 22 11:38:17 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 11:38:17 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 --
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 +--
 .../mr/steps/FactDistinctColumnsReducer.java| 240 ---
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 187 insertions(+), 175 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,10 +26,8 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -142,18 +140,4 @@ public class HadoopUtil {
 }
 }
 
-public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
-FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
-@Override
-public boolean accept(Path path) {
-return path.getName().startsWith(filter);
-}
-});
-
-if (fileStatus.length == 1) {
-return fileStatus[0].getPath();
-} else {
-return null;
-}
-}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index c34a904..696b22a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
+return getRealizationRootPath(jobId) + "/statistics";
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 602b4bb..0281539 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,16 +53,9 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
+String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
-String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
-
-String CFG_OUTPUT_COLUMN =

[14/15] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-22 Thread liyang 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/15ab71d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/15ab71d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/15ab71d3

Branch: refs/heads/master-cdh5.7
Commit: 15ab71d329fcbbb88eec75a66cf689ff8dc10890
Parents: 68e87f0
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 11:55:20 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/15ab71d3/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/15ab71d3/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index e02bf19..cbec4f6 100644
---

[06/14] kylin git commit: Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs"

2017-01-22 Thread liyang 
Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs"

This reverts commit 7de8aa1203a72bad105ed692f7100535939b03af.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7576a09f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7576a09f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7576a09f

Branch: refs/heads/master-hbase1.x
Commit: 7576a09f29b75f2eb80a1b5ca9c65fe5c81b1144
Parents: 7de8aa1
Author: shaofengshi 
Authored: Sun Jan 22 11:38:17 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 11:38:17 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 --
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 +--
 .../mr/steps/FactDistinctColumnsReducer.java| 240 ---
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 187 insertions(+), 175 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,10 +26,8 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -142,18 +140,4 @@ public class HadoopUtil {
 }
 }
 
-public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
-FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
-@Override
-public boolean accept(Path path) {
-return path.getName().startsWith(filter);
-}
-});
-
-if (fileStatus.length == 1) {
-return fileStatus[0].getPath();
-} else {
-return null;
-}
-}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index c34a904..696b22a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
+return getRealizationRootPath(jobId) + "/statistics";
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 602b4bb..0281539 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,16 +53,9 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
+String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
-String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
-
-String CFG_OUTPUT_COLUMN

[08/14] kylin git commit: Merge branch 'KYLIN-2394'

2017-01-22 Thread liyang 
Merge branch 'KYLIN-2394'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c4591a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c4591a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c4591a6

Branch: refs/heads/master-hbase1.x
Commit: 2c4591a609c8d8141c880722a3b8b282b785133e
Parents: d264339 d34f2d3
Author: Billy Liu 
Authored: Sun Jan 22 17:43:16 2017 +0800
Committer: Billy Liu 
Committed: Sun Jan 22 17:43:16 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3782 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2348 insertions(+), 1494 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2c4591a6/kylin-it/pom.xml
--

[03/14] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

2017-01-22 Thread liyang 
KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dfb5fac9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dfb5fac9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dfb5fac9

Branch: refs/heads/master-hbase1.x
Commit: dfb5fac9f0d9c13f1128206b73e683b515fe3d72
Parents: 61833d9
Author: Billy Liu 
Authored: Thu Jan 19 14:54:20 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 19 14:54:20 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3941 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2430 insertions(+), 1571 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b916df2..1b327fe 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -36,7 +36,16 @@
 
 org.apache.calcite
 calcite-core
+
+
+org.apache.calcite.avatica
+avatica-core
+
+
+
+
+org.apache.calcite.avatica
+avatica
 
 
-

[05/14] kylin git commit: KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs

2017-01-22 Thread liyang 
KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7de8aa12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7de8aa12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7de8aa12

Branch: refs/heads/master-hbase1.x
Commit: 7de8aa1203a72bad105ed692f7100535939b03af
Parents: c2229c9
Author: kangkaisen 
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: kangkaisen 
Committed: Sat Jan 21 23:19:50 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 ++-
 .../mr/steps/FactDistinctColumnsReducer.java| 240 +++
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 175 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
 }
 }
 
+public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+@Override
+public boolean accept(Path path) {
+return path.getName().startsWith(filter);
+}
+});
+
+if (fileStatus.length == 1) {
+return fileStatus[0].getPath();
+} else {
+return null;
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/statistics";
+return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
-String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
+String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+String CFG_OUTPUT_COLUMN = "column";
+String CFG_OUTPUT_DICT = "dict";
+String

[01/14] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0 [Forced Update!]

2017-01-22 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master-hbase1.x efbb2bab3 -> 15ab71d32 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index cd784f4..874ead6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -66,13 +66,15 @@
 
 
 
false
+
 
-
 
-
com.fasterxml.jackson.core:*
+com.google.protobuf:*
+commons-logging:*
+commons-codec:*
 
 
+
 
 
 org.apache.calcite
@@ -86,10 +88,6 @@
 org.apache.http
 
${shadeBase}.org.apache.http
 
-
-org.apache.commons
-
${shadeBase}.org.apache.commons
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index a1b9aef..8e69e68 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -209,7 +209,7 @@ public class KylinMeta extends MetaImpl {
 } catch (NoSuchFieldException e) {
 throw new RuntimeException(e);
 }
-columns.add(columnMetaData(name, index, field.getType()));
+columns.add(columnMetaData(name, index, field.getType(), true));
 fields.add(field);
 fieldNames.add(fieldName);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 080558b..80af108 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -43,6 +43,12 @@
 
 org.apache.kylin
 atopcalcite
+
+
+avatica-core
+org.apache.calcite.avatica
+
+
 
 
 org.apache.kylin
@@ -74,14 +80,6 @@
 org.apache.kylin
 kylin-query
 
-
-org.apache.calcite
-calcite-linq4j
-
-
-commons-cli
-commons-cli
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java 
b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index df6eb2f..05f615f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
 import java.io.File;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
+import java.sql.Driver;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.Statement;

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/pom.xml
--
diff --git a/pom.xml b/pom.xml
index ff4c4e8..57b7752 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,8 +112,8 @@
 1.8.9
 
 
-1.10.0
-1.8.0
+1.11.0
+1.9.0
 
 
 jacoco
@@ -450,15 +450,9 @@
 
 
 
-org.apache.calcite
-calcite-linq4j
-${calcite.version}
-
-
 org.apache.calcite.avatica
 avatica
 ${avatica.version}
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/query/pom.xml
--
diff --git a/query/pom.xml b/query/pom.xml
index 1dc05d1..6ab74a7 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -36,16 +36,18 @@
 
 org.apache.kylin
 atopcalcite
+
+
+

kylin git commit: Add logging info

2017-01-22 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2242 639f1f66a -> af149f562


Add logging info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af149f56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af149f56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af149f56

Branch: refs/heads/KYLIN-2242
Commit: af149f562461d1d022de5d3adf09c52bc9856fd0
Parents: 639f1f6
Author: shaofengshi 
Authored: Mon Jan 23 11:49:27 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 11:49:27 2017 +0800

--
 .../org/apache/kylin/source/kafka/job/MergeOffsetStep.java  | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/af149f56/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
index 9cadd72..914fca2 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.base.Preconditions;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
@@ -38,6 +39,7 @@ import org.slf4j.LoggerFactory;
 public class MergeOffsetStep extends AbstractExecutable {
 
 private static final Logger logger = 
LoggerFactory.getLogger(MergeOffsetStep.class);
+
 public MergeOffsetStep() {
 super();
 }
@@ -46,12 +48,15 @@ public class MergeOffsetStep extends AbstractExecutable {
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
 final CubeManager cubeManager = 
CubeManager.getInstance(context.getConfig());
 final CubeInstance cube = 
cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
-final CubeSegment segment = 
cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
+final String segmentId = 
CubingExecutableUtil.getSegmentId(this.getParams());
+final CubeSegment segment = cube.getSegmentById(segmentId);
 
+Preconditions.checkNotNull(segment, "Cube segment '" + segmentId + "' 
not found.");
 List mergingSegs = cube.getMergingSegments(segment);
 
-Collections.sort(mergingSegs);
+Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment 
not exist.");
 
+Collections.sort(mergingSegs);
 final CubeSegment first = mergingSegs.get(0);
 final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1);

[1/2] kylin git commit: KYLIN 2396 GUI for Percentile pre aggregation implementation

2017-01-22 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/master 6aaf6676b -> b56bc6220


KYLIN 2396 GUI for Percentile pre aggregation implementation

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b56bc622
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b56bc622
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b56bc622

Branch: refs/heads/master
Commit: b56bc622086495f7d0047d01af57efc919b2ecce
Parents: 28673a8
Author: luguosheng <550175...@qq.com>
Authored: Mon Jan 23 11:24:39 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 11:32:46 2017 +0800

--
 webapp/app/js/controllers/cubeMeasures.js  | 2 ++
 webapp/app/partials/cubeDesigner/measures.html | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 7c155fe..91c38f2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -355,6 +355,8 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   $scope.newMeasure.function.parameter.type= 'column';
   $scope.newMeasure.function.returntype = "extendedcolumn(100)";
   return;
+}else if($scope.newMeasure.function.expression=='PERCENTILE'){
+  $scope.newMeasure.function.parameter.type= 'column';
 }else{
   $scope.nextParameterInit();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index 4a48843..b7475b6 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -116,7 +116,7 @@
   
   Param Type
   
-  
   
   constant
+ ng-if="newMeasure.function.expression 
== 'COUNT'||newMeasure.function.expression == 'PERCENTILE'"
+ 
>{{newMeasure.function.parameter.type}}

[2/2] kylin git commit: KYLIN 2396 GUI for Percentile pre aggregation implementation

2017-01-22 Thread lidong 
KYLIN 2396 GUI for Percentile pre aggregation implementation

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28673a81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28673a81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28673a81

Branch: refs/heads/master
Commit: 28673a8126ec2ecff8c125c0ecf887ce2565362c
Parents: 6aaf667
Author: chenzhx <346839...@qq.com>
Authored: Fri Jan 20 18:00:20 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 11:32:46 2017 +0800

--
 webapp/app/js/controllers/cubeMeasures.js  | 18 +++
 webapp/app/js/model/cubeConfig.js  |  2 +-
 webapp/app/partials/cubeDesigner/measures.html | 54 -
 3 files changed, 40 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index c8548ce..7c155fe 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -80,6 +80,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   $scope.updateMeasureStatus.editIndex = index;
 }
 $scope.nextParameters = [];
+$scope.measureParamValueColumn=$scope.getCommonMetricColumns();
 $scope.newMeasure = (!!measure)? jQuery.extend(true, 
{},measure):CubeDescModel.createMeasure();
 if(!!measure && measure.function.parameter.next_parameter){
   $scope.nextPara.value = measure.function.parameter.next_parameter.value;
@@ -93,6 +94,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 }else{
   $scope.newMeasure.showDim=false;
 }
+$scope.measureParamValueUpdate();
 if($scope.newMeasure.function.expression=="TOP_N"){
   $scope.convertedColumns=[];
   if($scope.newMeasure.function.configuration==null){
@@ -326,6 +328,19 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   return false;
 }
   }
+
+  $scope.measureParamValueUpdate = function(){
+if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && 
$scope.newMeasure.showDim==true){
+   $scope.measureParamValueColumn=$scope.getAllModelDimMeasureColumns();
+}
+if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && 
$scope.newMeasure.showDim==false){
+   $scope.measureParamValueColumn=$scope.getCommonMetricColumns();
+}
+if($scope.newMeasure.function.expression == 'EXTENDED_COLUMN'){
+  $scope.measureParamValueColumn=$scope.getExtendedHostColumn();
+}
+  }
+
   //map right return type for param
   $scope.measureReturnTypeUpdate = function(){
 
@@ -393,6 +408,9 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 case "COUNT":
   $scope.newMeasure.function.returntype = "bigint";
   break;
+case "PERCENTILE":
+  $scope.newMeasure.function.returntype = "percentile(100)";
+  break;
 default:
   $scope.newMeasure.function.returntype = "";
   break;

http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/model/cubeConfig.js
--
diff --git a/webapp/app/js/model/cubeConfig.js 
b/webapp/app/js/model/cubeConfig.js
index 87413dc..35af1fc 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -20,7 +20,7 @@ KylinApp.constant('cubeConfig', {
 
   //~ Define metadata & class
   measureParamType: ['column', 'constant'],
-  measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 
'RAW','EXTENDED_COLUMN'],
+  measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 
'RAW','EXTENDED_COLUMN','PERCENTILE'],
   dimensionDataTypes: ["string", "tinyint", "int", "bigint", "date"],
   cubePartitionTypes: ['APPEND'],
   joinTypes: [

http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index 4d2d8ab..4a48843 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -104,7 +104,7 @@
   
   
   
   
@@ -144,35 +144,23 @@
 
 
   
-

[kylin] Git Push Summary

2017-01-22 Thread billyliu 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2394 [deleted] d34f2d3e2

[2/5] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

2017-01-22 Thread billyliu 
http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index e0782ce..171e299 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -16,24 +16,6 @@
  */
 package org.apache.calcite.sql2rel;
 
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
 import org.apache.calcite.avatica.util.Spaces;
 import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.plan.Convention;
@@ -85,7 +67,6 @@ import org.apache.calcite.rel.stream.Delta;
 import org.apache.calcite.rel.stream.LogicalDelta;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -116,6 +97,7 @@ import org.apache.calcite.sql.SqlCallBinding;
 import org.apache.calcite.sql.SqlDataTypeSpec;
 import org.apache.calcite.sql.SqlDelete;
 import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainFormat;
 import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.calcite.sql.SqlFunction;
 import org.apache.calcite.sql.SqlIdentifier;
@@ -177,17 +159,37 @@ import org.apache.calcite.util.NumberUtil;
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Util;
 import org.apache.calcite.util.trace.CalciteTrace;
-import org.slf4j.Logger;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+
+import org.slf4j.Logger;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
 
 /*
  * The code has synced with calcite. Hope one day, we could remove the 
hardcode override point.
@@ -209,42 +211,56 @@ import com.google.common.collect.Sets;
 public class SqlToRelConverter {
 //~ Static fields/initializers 
-
 
+protected static final Logger SQL2REL_LOGGER =
+CalciteTrace.getSqlToRelTracer();
+
+private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
 /** Size of the smallest IN list that will be converted to a semijoin to a
  * static table. */
-public static final int DEFAULT_IN_SUBQUERY_THRESHOLD = 20;
-protected static final Logger SQL2REL_LOGGER = 
CalciteTrace.getSqlToRelTracer();
-private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+
+@Deprecated // to be removed before 2.0
+public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
+DEFAULT_IN_SUB_QUERY_THRESHOLD;
 
 //~ Instance fields 

-public final SqlToRelConverter.Config config;
-public final RelOptTable.ViewExpander viewExpander;
+
 protected final SqlValidator validator;
 protected final RexBuilder rexBuilder;
 protected final Prepare.CatalogReader catalogReader;
 protected final RelOptCluster cluster;
+private DefaultValueFactory defaultValueFactory;
+private SubQueryConverter subQueryConverter;
 protected final List leaves = new ArrayList<>();
-protected final RelDataTypeFactory typeFactory;
 private final List dynamicParamSqlNodes = new 
ArrayList<>();

[1/5] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

2017-01-22 Thread billyliu 
Repository: kylin
Updated Branches:
  refs/heads/master d264339b1 -> 2c4591a60


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index cd784f4..874ead6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -66,13 +66,15 @@
 
 
 
false
+
 
-
 
-
com.fasterxml.jackson.core:*
+com.google.protobuf:*
+commons-logging:*
+commons-codec:*
 
 
+
 
 
 org.apache.calcite
@@ -86,10 +88,6 @@
 org.apache.http
 
${shadeBase}.org.apache.http
 
-
-org.apache.commons
-
${shadeBase}.org.apache.commons
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index a1b9aef..8e69e68 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -209,7 +209,7 @@ public class KylinMeta extends MetaImpl {
 } catch (NoSuchFieldException e) {
 throw new RuntimeException(e);
 }
-columns.add(columnMetaData(name, index, field.getType()));
+columns.add(columnMetaData(name, index, field.getType(), true));
 fields.add(field);
 fieldNames.add(fieldName);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 080558b..80af108 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -43,6 +43,12 @@
 
 org.apache.kylin
 atopcalcite
+
+
+avatica-core
+org.apache.calcite.avatica
+
+
 
 
 org.apache.kylin
@@ -74,14 +80,6 @@
 org.apache.kylin
 kylin-query
 
-
-org.apache.calcite
-calcite-linq4j
-
-
-commons-cli
-commons-cli
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java 
b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index df6eb2f..05f615f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
 import java.io.File;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
+import java.sql.Driver;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.Statement;

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/pom.xml
--
diff --git a/pom.xml b/pom.xml
index ff4c4e8..57b7752 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,8 +112,8 @@
 1.8.9
 
 
-1.10.0
-1.8.0
+1.11.0
+1.9.0
 
 
 jacoco
@@ -450,15 +450,9 @@
 
 
 
-org.apache.calcite
-calcite-linq4j
-${calcite.version}
-
-
 org.apache.calcite.avatica
 avatica
 ${avatica.version}
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/query/pom.xml
--
diff --git a/query/pom.xml b/query/pom.xml
index 1dc05d1..6ab74a7 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -36,16 +36,18 @@
 
 org.apache.kylin
 atopcalcite
+
+
+

[3/5] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

2017-01-22 Thread billyliu 
KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dfb5fac9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dfb5fac9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dfb5fac9

Branch: refs/heads/master
Commit: dfb5fac9f0d9c13f1128206b73e683b515fe3d72
Parents: 61833d9
Author: Billy Liu 
Authored: Thu Jan 19 14:54:20 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 19 14:54:20 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3941 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2430 insertions(+), 1571 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b916df2..1b327fe 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -36,7 +36,16 @@
 
 org.apache.calcite
 calcite-core
+
+
+org.apache.calcite.avatica
+avatica-core
+
+
+
+
+org.apache.calcite.avatica
+avatica
 
 
-

[5/5] kylin git commit: Merge branch 'KYLIN-2394'

2017-01-22 Thread billyliu 
Merge branch 'KYLIN-2394'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c4591a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c4591a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c4591a6

Branch: refs/heads/master
Commit: 2c4591a609c8d8141c880722a3b8b282b785133e
Parents: d264339 d34f2d3
Author: Billy Liu 
Authored: Sun Jan 22 17:43:16 2017 +0800
Committer: Billy Liu 
Committed: Sun Jan 22 17:43:16 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3782 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2348 insertions(+), 1494 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2c4591a6/kylin-it/pom.xml
--

[4/5] kylin git commit: KYLIN-2394 merge CALCITE-1592

2017-01-22 Thread billyliu 
KYLIN-2394 merge CALCITE-1592


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34f2d3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34f2d3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34f2d3e

Branch: refs/heads/master
Commit: d34f2d3e2bf691c55077a9a76a89217ae915bbdd
Parents: dfb5fac
Author: Billy Liu 
Authored: Fri Jan 20 15:42:37 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 15:42:37 2017 +0800

--
 .../calcite/sql2rel/SqlToRelConverter.java  | 51 +---
 1 file changed, 23 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d34f2d3e/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 171e299..cc9c9a1 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -935,19 +935,21 @@ public class SqlToRelConverter {
 /**
  * Push down all the NOT logical operators into any IN/NOT IN operators.
  *
+ * @param scope Scope where {@code sqlNode} occurs
  * @param sqlNode the root node from which to look for NOT operators
  * @return the transformed SqlNode representation with NOT pushed down.
  */
-private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+private static SqlNode pushDownNotForIn(SqlValidatorScope scope,
+SqlNode sqlNode) {
 if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
 SqlCall sqlCall = (SqlCall) sqlNode;
 if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
 || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
 SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
 for (int i = 0; i < sqlOperands.length; i++) {
-sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+sqlOperands[i] = pushDownNotForIn(scope, sqlOperands[i]);
 }
-return sqlNode;
+return reg(scope, sqlNode);
 } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
 SqlNode childNode = sqlCall.operand(0);
 assert childNode instanceof SqlCall;
@@ -956,48 +958,34 @@ public class SqlToRelConverter {
 SqlNode[] andOperands = childSqlCall.getOperands();
 SqlNode[] orOperands = new SqlNode[andOperands.length];
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-andOperands[i]);
+orOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]));
 }
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] = pushDownNotForIn(orOperands[i]);
+orOperands[i] = pushDownNotForIn(scope, orOperands[i]);
 }
-return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
-orOperands[0], orOperands[1]);
+return reg(scope, 
SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], 
orOperands[1]));
 } else if (childSqlCall.getOperator() == 
SqlStdOperatorTable.OR) {
 SqlNode[] orOperands = childSqlCall.getOperands();
 SqlNode[] andOperands = new SqlNode[orOperands.length];
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-orOperands[i]);
+andOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]));
 }
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] = pushDownNotForIn(andOperands[i]);
+andOperands[i] = pushDownNotForIn(scope, 
andOperands[i]);
 }
-return 
SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
-

kylin git commit: KYLIN-2217 use column’s identity as path name and add more logging info [Forced Update!]

2017-01-22 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2242 71994afb3 -> 639f1f66a (forced update)


KYLIN-2217 use column’s identity as path name and add more logging info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/639f1f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/639f1f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/639f1f66

Branch: refs/heads/KYLIN-2242
Commit: 639f1f66aef95a2a22bfd61a2e3fcf94646f041a
Parents: 7de8aa1
Author: shaofengshi 
Authored: Sun Jan 22 09:43:10 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 16:56:52 2017 +0800

--
 .../java/org/apache/kylin/common/util/HadoopUtil.java|  4 
 .../apache/kylin/cube/cli/DictionaryGeneratorCLI.java|  3 +++
 .../org/apache/kylin/engine/mr/SortedColumnDFSFile.java  |  4 
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java |  2 +-
 .../kylin/engine/mr/steps/CreateDictionaryJob.java   |  9 +++--
 .../engine/mr/steps/FactDistinctColumnsReducer.java  | 11 +++
 .../engine/mr/steps/UpdateCubeInfoAfterBuildStep.java|  4 ++--
 .../hive/cardinality/HiveColumnCardinalityJob.java   |  3 +++
 8 files changed, 31 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/639f1f66/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..f242515 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -143,6 +143,10 @@ public class HadoopUtil {
 }
 
 public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+if (fs.exists(baseDir) == false) {
+return null;
+}
+
 FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
 @Override
 public boolean accept(Path path) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/639f1f66/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 3e1ab0d..2e5a38e 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -63,11 +63,14 @@ public class DictionaryGeneratorCLI {
 if (dictProvider != null) {
 Dictionary dict = dictProvider.getDictionary(col);
 if (dict != null) {
+logger.debug("Dict for '" + col.getName() + "' has already 
been built, save it");
 cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);
 } else {
+logger.debug("Dict for '" + col.getName() + "' not 
pre-built, build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 } else {
+logger.debug("Dict for '" + col.getName() + "' not pre-built, 
build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/639f1f66/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
index d3f5cdc..f396b5a 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
@@ -128,4 +128,8 @@ public class SortedColumnDFSFile implements ReadableTable {
 return comparator;
 }
 
+@Override
+public String toString() {
+return dfsPath;
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/639f1f66/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java

kylin git commit: minor, remove unused import

2017-01-22 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2242 73ec2b956 -> 71994afb3


minor, remove unused import


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71994afb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71994afb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71994afb

Branch: refs/heads/KYLIN-2242
Commit: 71994afb315ad88a5d79e551203cf19148730ed3
Parents: 73ec2b9
Author: shaofengshi 
Authored: Sun Jan 22 16:02:13 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 16:02:13 2017 +0800

--
 .../kylin/source/hive/cardinality/HiveColumnCardinalityJob.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/71994afb/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
index 2b631ff..f439ccb 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
@@ -34,7 +34,6 @@ import 
org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
 import org.apache.kylin.engine.mr.MRUtil;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.steps.CreateDictionaryJob;
 import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.TableDesc;