[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c47f4ad5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c47f4ad5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c47f4ad5

Branch: refs/heads/master-cdh5.7
Commit: c47f4ad5767e7c9604b78bbae798805ce22e02fe
Parents: b0a406f
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Thu Jan 12 12:02:26 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 62978db..e46cbe1 100644
--- 

[25/50] [abbrv] kylin git commit: minor, enable jacoco plugin

[lidong]

minor, enable jacoco plugin


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68ff75de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68ff75de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68ff75de

Branch: refs/heads/master-cdh5.7
Commit: 68ff75de260e6e93197103c0fd06009cf73c8e21
Parents: e46d699
Author: Billy Liu 
Authored: Mon Jan 9 00:29:57 2017 +0800
Committer: Billy Liu 
Committed: Mon Jan 9 00:29:57 2017 +0800

--
 pom.xml | 47 +++
 1 file changed, 31 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/68ff75de/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 22681e8..d883eca 100644
--- a/pom.xml
+++ b/pom.xml
@@ -922,19 +922,7 @@
 
 org.jacoco
 jacoco-maven-plugin
-0.7.7.201606060606
-
-${sonar.jacoco.reportPath}
-true
-
-
-
-agent
-
-prepare-agent
-
-
-
+0.7.8
 
 
 org.apache.maven.plugins
@@ -1079,6 +1067,35 @@
 
 
 
+org.jacoco
+jacoco-maven-plugin
+
+true
+
+${sonar.jacoco.reportPath}
+
+
+
+
+pre-test
+
+prepare-agent
+
+
+
surefireArgLine
+
+
+
+post-test
+test
+
+report
+
+
+
+
+
+
 org.apache.maven.plugins
 maven-surefire-plugin
 2.19.1
@@ -1101,11 +1118,9 @@
 
file:${project.basedir}/../build/conf/kylin-tools-log4j.properties
 
 
-
-javaagent:${project.build.testOutputDirectory}/jamm.jar 
${argLine}
+
-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} 
${surefireArgLine}
 
 
-
-
 
 
 

[35/50] [abbrv] kylin git commit: modify default partition setting

[lidong]

modify default partition setting


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2cf52b49
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2cf52b49
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2cf52b49

Branch: refs/heads/master-cdh5.7
Commit: 2cf52b49ed9a1158cb2ec0ded6d907f90ee024fe
Parents: b9a69ef
Author: shaofengshi 
Authored: Thu Jan 5 19:56:10 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 build/conf/kylin.properties | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2cf52b49/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index bd0bbd4..54430f0 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -137,13 +137,13 @@ kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
 #kylin.engine.spark.properties-file=
 
 # Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=200
+kylin.engine.spark.rdd-partition-cut-mb=50
 
 # Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=1
+kylin.engine.spark.min-partition=10
 
 # Max partition numbers of rdd
-kylin.engine.spark.max-partition=500
+kylin.engine.spark.max-partition=5000
 
 ### CUBE | DICTIONARY ###
 

[03/50] [abbrv] kylin git commit: KYLIN-2333 revert the changes as there is the case that “mandatory only combination is allowed”

[lidong]

KYLIN-2333 revert the changes as there is the case that “mandatory only 
combination is allowed”


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/df5faf37
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/df5faf37
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/df5faf37

Branch: refs/heads/master-cdh5.7
Commit: df5faf37cd3186376c28018a5a5a7e42bcea1db9
Parents: 4b5257a
Author: shaofengshi 
Authored: Thu Jan 5 15:14:37 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 5 15:14:37 2017 +0800

--
 .../java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java| 3 +--
 .../java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java   | 2 +-
 .../main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java   | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/df5faf37/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 8fbecd3..456f615 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -65,12 +65,11 @@ public class BatchCubingJobBuilder extends 
JobBuilderSupport {
 // Phase 3: Build Cube
 RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey();
 final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel();
-final int totalRowkeyColumnsCount = 
rowKeyDesc.getRowKeyColumns().length;
 final String[] cuboidOutputTempPath = 
getCuboidOutputPaths(cuboidRootPath, groupRowkeyColumnsCount);
 // base cuboid step
 result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
 // n dim cuboid steps
-for (int i = 1; i < groupRowkeyColumnsCount; i++) {
+for (int i = 1; i <= groupRowkeyColumnsCount; i++) {
 result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 
1], cuboidOutputTempPath[i], i));
 }
 outputSide.addStepPhase3_BuildCube(result, cuboidRootPath);

http://git-wip-us.apache.org/repos/asf/kylin/blob/df5faf37/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 9229a75..1e3f078 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -81,7 +81,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport 
{
 // base cuboid step
 result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
 // n dim cuboid steps
-for (int i = 1; i < maxLevel; i++) {
+for (int i = 1; i <= maxLevel; i++) {
 result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 
1], cuboidOutputTempPath[i], i, jobId));
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/df5faf37/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 23fce2b..122b8ba 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -194,7 +194,7 @@ public class JobBuilderSupport {
 
 public String[] getCuboidOutputPaths(String cuboidRootPath, int levels) {
 String[] paths = new String[levels];
-for (int i = 0; i < levels; i++) {
+for (int i = 0; i <= levels; i++) {
 if (i == 0) {
 paths[i] = cuboidRootPath + "base_cuboid";
 } else {

[18/50] [abbrv] kylin git commit: KYLIN-2337 minor improvement, still give good error info

[lidong]

KYLIN-2337 minor improvement, still give good error info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/122b15a2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/122b15a2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/122b15a2

Branch: refs/heads/master-cdh5.7
Commit: 122b15a2a5fb7385e64839edfb9cbdaec5fde841
Parents: 7509bee
Author: Yang Li 
Authored: Sat Jan 7 09:38:06 2017 +0800
Committer: Yang Li 
Committed: Sat Jan 7 09:38:06 2017 +0800

--
 .../kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/122b15a2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
index 6a7e713..c5c95a6 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
@@ -126,7 +126,8 @@ public class SortedIteratorMergerWithLimit extends SortedIt
 
 //TODO: remove this check when validated
 if (last != null) {
-Preconditions.checkState(comparator.compare(last, fetched) <= 
0, "Not sorted!");
+if (comparator.compare(last, fetched) > 0)
+throw new IllegalStateException("Not sorted! last: " + 
last + " fetched: " + fetched);
 }
 
 last = fetched;

[19/50] [abbrv] kylin git commit: fix NPE bug

[lidong]

fix NPE bug

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5a18af04
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5a18af04
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5a18af04

Branch: refs/heads/master-cdh5.7
Commit: 5a18af04ca40aaa6509cdf7c39b71b17dc5eb4a3
Parents: 122b15a
Author: xiefan46 <958034...@qq.com>
Authored: Thu Jan 5 23:29:43 2017 +0800
Committer: shaofengshi 
Committed: Sat Jan 7 10:18:20 2017 +0800

--
 .../src/main/java/org/apache/kylin/dict/CacheDictionary.java | 8 ++--
 1 file changed, 2 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5a18af04/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
index 1e260b2..d7ed6bd 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
@@ -29,8 +29,6 @@ import java.util.concurrent.ConcurrentHashMap;
 public abstract class CacheDictionary extends Dictionary {
 private static final long serialVersionUID = 1L;
 
-protected transient boolean enableValueCache = false;
-
 private transient SoftReference valueToIdCache;
 
 private transient SoftReference idToValueCache;
@@ -46,7 +44,7 @@ public abstract class CacheDictionary extends 
Dictionary {
 //value --> id
 @Override
 protected final int getIdFromValueImpl(T value, int roundingFlag) {
-if (enableValueCache && roundingFlag == 0) {
+if (this.valueToIdCache != null && roundingFlag == 0) {
 Map cache = valueToIdCache.get(); // SoftReference to skip cache 
gracefully when short of memory
 if (cache != null) {
 Integer id;
@@ -66,7 +64,7 @@ public abstract class CacheDictionary extends 
Dictionary {
 //id --> value
 @Override
 protected final T getValueFromIdImpl(int id) {
-if (enableValueCache) {
+if (this.idToValueCache != null) {
 Object[] cache = idToValueCache.get();
 if (cache != null) {
 int seq = calcSeqNoFromId(id);
@@ -91,7 +89,6 @@ public abstract class CacheDictionary extends 
Dictionary {
 }
 
 public final void enableCache() {
-this.enableValueCache = true;
 if (this.valueToIdCache == null)
 this.valueToIdCache = new SoftReference<>(new ConcurrentHashMap());
 if (this.idToValueCache == null)
@@ -99,7 +96,6 @@ public abstract class CacheDictionary extends 
Dictionary {
 }
 
 public final void disableCache() {
-this.enableValueCache = false;
 this.valueToIdCache = null;
 this.idToValueCache = null;
 }

[48/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
 FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 String localCoprocessorJar;
 if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
 public static void deployCoprocessor(HTableDescriptor tableDesc) {
 try {
 initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with 
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed 
with coprocessor.");
 
 } catch (Exception ex) {
-logger.error("Error deploying coprocessor on " + 
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " + 
tableDesc.getTableName(), ex);
 logger.error("Will try creating the table without coprocessor.");
 }
 }
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
 desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
 }
 
-public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, 
Path hdfsCoprocessorJar) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 HTableDescriptor desc = 
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
 
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
 logger.info("reset coprocessor on " + tableName);
 
 logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
 
 while (desc.hasCoprocessor(CubeObserverClassOld2)) {
 desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
 desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
 }
 
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
 
 logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
 
 return true;
 }
 
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
+private static List resetCoprocessorOnHTables(final Admin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
 List processedTables = Collections.synchronizedList(new 
ArrayList());
 ExecutorService coprocessorPool = 
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
 CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
 
 private static class ResetCoprocessorWorker implements Runnable {
 private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
 private final Path hdfsCoprocessorJar;
 private final String tableName;
 private final List processedTables;
 
-public ResetCoprocessorWorker(CountDownLatch countDownLatch, 
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List 
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch 

[32/50] [abbrv] kylin git commit: KYLIN-2331 by layer spark cubing

[lidong]

KYLIN-2331 by layer spark cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b7d87bb4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b7d87bb4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b7d87bb4

Branch: refs/heads/master-cdh5.7
Commit: b7d87bb4e6c4de484d910e9af04965392da518e5
Parents: 75625ae
Author: shaofengshi 
Authored: Sun Dec 25 15:59:16 2016 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 assembly/pom.xml|   5 +-
 .../main/config/assemblies/source-assembly.xml  |   3 -
 .../java/org/apache/kylin/job/DeployUtil.java   |   1 -
 build/conf/kylin-spark-conf.properties  |  26 ++
 build/conf/kylin.properties |  20 +
 .../apache/kylin/common/KylinConfigBase.java|  64 ++-
 .../kylin/common/persistence/ResourceStore.java |  35 ++
 .../org/apache/kylin/common/util/Array.java |   2 +-
 .../apache/kylin/common/util/SplittedBytes.java |   2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   2 +-
 .../kylin/cube/common/RowKeySplitter.java   |  15 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java|  27 +-
 .../kylin/cube/cuboid/CuboidScheduler.java  |  15 +-
 .../cube/inmemcubing/InMemCubeBuilder.java  |   2 +
 .../kylin/cube/kv/AbstractRowKeyEncoder.java|   6 +-
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |   7 +-
 .../apache/kylin/cube/kv/RowKeyColumnIO.java|   2 +-
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java |  19 +-
 .../kylin/cube/kv/RowKeyEncoderProvider.java|   2 +-
 .../kylin/cube/model/AggregationGroup.java  |   2 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |   2 +-
 .../cube/model/CubeJoinedFlatTableEnrich.java   |   6 +-
 .../apache/kylin/cube/model/DictionaryDesc.java |   4 +-
 .../apache/kylin/cube/model/DimensionDesc.java  |  14 +-
 .../kylin/cube/model/HBaseColumnDesc.java   |  11 +-
 .../kylin/cube/model/HBaseColumnFamilyDesc.java |   9 +-
 .../kylin/cube/model/HBaseMappingDesc.java  |  10 +-
 .../apache/kylin/cube/model/HierarchyDesc.java  |   2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |   2 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java |  18 +-
 .../org/apache/kylin/cube/model/SelectRule.java |   2 +-
 .../apache/kylin/gridtable/UnitTestSupport.java |  41 +-
 .../gridtable/AggregationCacheMemSizeTest.java  |   6 +-
 .../gridtable/AggregationCacheSpillTest.java|   5 +-
 .../kylin/gridtable/SimpleGridTableTest.java|  11 +-
 .../gridtable/SimpleInvertedIndexTest.java  |   3 +-
 .../metadata/measure/MeasureCodecTest.java  |   6 +-
 .../apache/kylin/dict/AppendTrieDictionary.java |   6 +-
 .../org/apache/kylin/dict/NumberDictionary.java |   2 +-
 .../apache/kylin/dict/NumberDictionary2.java|   2 +-
 .../dict/NumberDictionaryForestBuilder.java |   4 +-
 .../apache/kylin/dict/StringBytesConverter.java |   2 +-
 .../kylin/job/execution/ExecutableManager.java  |  21 +-
 .../kylin/dimension/AbstractDateDimEnc.java |  12 +-
 .../apache/kylin/dimension/BooleanDimEnc.java   |   4 +-
 .../kylin/dimension/DictionaryDimEnc.java   |  21 +-
 .../apache/kylin/dimension/FixedLenDimEnc.java  |   4 +-
 .../kylin/dimension/FixedLenHexDimEnc.java  |   4 +-
 .../org/apache/kylin/dimension/IntDimEnc.java   |   4 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   4 +-
 .../kylin/dimension/OneMoreByteVLongDimEnc.java |  14 +-
 .../kylin/measure/BufferedMeasureCodec.java |  10 +-
 .../apache/kylin/measure/MeasureAggregator.java |   2 +
 .../kylin/measure/MeasureAggregators.java   |  18 +
 .../org/apache/kylin/measure/MeasureCodec.java  |   8 +-
 .../apache/kylin/measure/MeasureIngester.java   |  12 +-
 .../org/apache/kylin/measure/MeasureType.java   |  12 +-
 .../measure/basic/BigDecimalMaxAggregator.java  |  14 +
 .../measure/basic/BigDecimalMinAggregator.java  |  28 +-
 .../measure/basic/BigDecimalSumAggregator.java  |  10 +
 .../kylin/measure/basic/DoubleIngester.java |  19 +-
 .../measure/basic/DoubleMaxAggregator.java  |  20 +-
 .../measure/basic/DoubleMinAggregator.java  |  20 +-
 .../measure/basic/DoubleSumAggregator.java  |  18 +-
 .../kylin/measure/basic/LongIngester.java   |  19 +-
 .../kylin/measure/basic/LongMaxAggregator.java  |  20 +-
 .../kylin/measure/basic/LongMinAggregator.java  |  20 +-
 .../kylin/measure/basic/LongSumAggregator.java  |  18 +-
 .../kylin/measure/bitmap/BitmapAggregator.java  |  13 +
 .../kylin/measure/bitmap/BitmapCounter.java |   2 +-
 .../kylin/measure/bitmap/BitmapMeasureType.java |   5 +
 .../ExtendedColumnMeasureType.java  |  15 +
 .../kylin/measure/hllc/DenseRegister.java   |   2 +-
 .../kylin/measure/hllc/HLLCAggregator.java  |   7 +
 

[22/50] [abbrv] kylin git commit: KYLIN-2353 Serialize BitmapCounter with distinct count

[lidong]

KYLIN-2353 Serialize BitmapCounter with distinct count

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/731a53a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/731a53a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/731a53a5

Branch: refs/heads/master-cdh5.7
Commit: 731a53a51a32fb1314b4e1282f158e3b7ed819a5
Parents: aa57446
Author: kangkaisen 
Authored: Sat Dec 31 20:28:30 2016 +0800
Committer: Yang Li 
Committed: Sat Jan 7 19:08:37 2017 +0800

--
 .../kylin/measure/bitmap/BitmapCounter.java | 105 ---
 .../bitmap/BitmapDistinctCountAggFunc.java  |   2 +-
 .../kylin/measure/bitmap/BitmapSerializer.java  |  15 +--
 3 files changed, 70 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/731a53a5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index aeb14ba..a18ac4a 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -35,6 +35,8 @@ public class BitmapCounter implements 
Comparable {
 
 private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
 private final int VERSION = 2;
+private Integer count;
+private ByteBuffer buffer;
 
 public BitmapCounter() {
 }
@@ -43,18 +45,41 @@ public class BitmapCounter implements 
Comparable {
 merge(another);
 }
 
+private MutableRoaringBitmap getBitmap() {
+if (!bitmap.isEmpty()) {
+return bitmap;
+}
+
+if (buffer != null) {
+int version = buffer.getInt();
+int size = buffer.getInt();
+count = buffer.getInt();
+
+try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(buffer))) {
+bitmap.deserialize(is);
+} catch (IOException e) {
+throw new RuntimeException("deserialize bitmap failed!");
+}
+
+buffer = null;
+}
+
+return bitmap;
+}
+
 public void clear() {
-bitmap.clear();
+getBitmap().clear();
 }
 
 public BitmapCounter clone() {
 BitmapCounter newCounter = new BitmapCounter();
-newCounter.bitmap = bitmap.clone();
+newCounter.bitmap = getBitmap().clone();
 return newCounter;
 }
 
 public void add(int value) {
-bitmap.add(value);
+getBitmap().add(value);
+count = getBitmap().getCardinality();
 }
 
 public void add(byte[] value) {
@@ -77,34 +102,43 @@ public class BitmapCounter implements 
Comparable {
 }
 
 public void merge(BitmapCounter another) {
-this.bitmap.or(another.bitmap);
+getBitmap().or(another.getBitmap());
+count = getBitmap().getCardinality();
 }
 
 public void intersect(BitmapCounter another) {
-this.bitmap.and(another.bitmap);
+getBitmap().and(another.getBitmap());
+count = getBitmap().getCardinality();
 }
 
-public long getCount() {
-return this.bitmap.getCardinality();
+public int getCount() {
+if (count != null) {
+return count;
+}
+
+return getBitmap().getCardinality();
 }
 
 public int getMemBytes() {
-return this.bitmap.getSizeInBytes();
+return getBitmap().getSizeInBytes();
 }
 
 public Iterator iterator() {
-return bitmap.iterator();
+return getBitmap().iterator();
 }
 
 public void writeRegisters(ByteBuffer out) throws IOException {
 ByteArrayOutputStream bos = new ByteArrayOutputStream();
 DataOutputStream dos = new DataOutputStream(bos);
+MutableRoaringBitmap bitmap = getBitmap();
 bitmap.runOptimize();
 bitmap.serialize(dos);
 dos.close();
 ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray());
+
 out.putInt(VERSION);
-out.putInt(bos.size() + 4 + 4);
+out.putInt(bos.size() + 4 + 4 + 4);
+out.putInt(getCount());
 out.put(bb);
 }
 
@@ -116,13 +150,30 @@ public class BitmapCounter implements 
Comparable {
 if (version == VERSION) {
 @SuppressWarnings("unused")
 int size = in.getInt();
+count = in.getInt();
+in.position(mark);
+buffer = cloneBuffer(in, size);
 } else {
 in.position(mark);
+   

[38/50] [abbrv] kylin git commit: minor, move duplicated ByteBufferBackedInputStream to common package

[lidong]

minor, move duplicated ByteBufferBackedInputStream to common package


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0aadb12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0aadb12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0aadb12

Branch: refs/heads/master-cdh5.7
Commit: b0aadb12d1e25005445eda0d912c63a941ad929a
Parents: 3a36c66
Author: gaodayue 
Authored: Tue Jan 10 11:51:38 2017 +0800
Committer: gaodayue 
Committed: Tue Jan 10 11:52:04 2017 +0800

--
 .../util/ByteBufferBackedInputStream.java   | 61 
 .../kylin/measure/bitmap/BitmapCounter.java | 38 +---
 .../measure/bitmap/BitmapAggregatorTest.java|  2 +-
 .../kylin/measure/bitmap/BitmapCounterTest.java |  2 +-
 .../measure/bitmap/BitmapSerializerTest.java|  2 +-
 .../source/kafka/TimedJsonStreamParser.java |  2 +-
 .../kafka/util/ByteBufferBackedInputStream.java | 52 -
 7 files changed, 66 insertions(+), 93 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
 
b/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
new file mode 100644
index 000..7332ff2
--- /dev/null
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+/**
+ * Utility to view content available in a {@link ByteBuffer} as a {@link 
InputStream}.
+ *
+ * Not thread-safe
+ */
+public class ByteBufferBackedInputStream extends InputStream {
+private final ByteBuffer buffer;
+
+public ByteBufferBackedInputStream(ByteBuffer buffer) {
+this.buffer = buffer;
+}
+
+@Override
+public int available() throws IOException {
+return buffer.remaining();
+}
+
+@Override
+public int read() throws IOException {
+return buffer.hasRemaining() ? (buffer.get() & 0xFF) : -1;
+}
+
+@Override
+public int read(byte[] b, int off, int len) throws IOException {
+if (len == 0) {
+return 0;
+}
+
+if (!buffer.hasRemaining()) {
+return -1;
+}
+
+len = Math.min(buffer.remaining(), len);
+buffer.get(b, off, len);
+return len;
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index 43ee506..19d7f5d 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -22,10 +22,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 
+import org.apache.kylin.common.util.ByteBufferBackedInputStream;
 import org.roaringbitmap.buffer.MutableRoaringBitmap;
 
 /**
@@ -84,18 +84,6 @@ public class BitmapCounter implements 
Comparable, java.io.Seriali
 count = null;
 }
 
-public void add(byte[] value) {
-add(value, 0, value.length);
-}
-
-public void add(byte[] value, int offset, int length) {
-if (value == null || length == 0) {
-return;
-}
-
-add(new 

[23/50] [abbrv] kylin git commit: KYLIN-2353 minor code review

[lidong]

KYLIN-2353 minor code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56daf6d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56daf6d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56daf6d5

Branch: refs/heads/master-cdh5.7
Commit: 56daf6d592ae454b3bca97c4a7a948781485befa
Parents: 731a53a
Author: Yang Li 
Authored: Sat Jan 7 19:30:27 2017 +0800
Committer: Yang Li 
Committed: Sat Jan 7 19:30:27 2017 +0800

--
 .../org/apache/kylin/measure/bitmap/BitmapCounter.java| 10 ++
 1 file changed, 6 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56daf6d5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index a18ac4a..a632b0d 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -51,7 +51,9 @@ public class BitmapCounter implements 
Comparable {
 }
 
 if (buffer != null) {
+@SuppressWarnings("unused")
 int version = buffer.getInt();
+@SuppressWarnings("unused")
 int size = buffer.getInt();
 count = buffer.getInt();
 
@@ -79,7 +81,7 @@ public class BitmapCounter implements 
Comparable {
 
 public void add(int value) {
 getBitmap().add(value);
-count = getBitmap().getCardinality();
+count = null;
 }
 
 public void add(byte[] value) {
@@ -103,12 +105,12 @@ public class BitmapCounter implements 
Comparable {
 
 public void merge(BitmapCounter another) {
 getBitmap().or(another.getBitmap());
-count = getBitmap().getCardinality();
+count = null;
 }
 
 public void intersect(BitmapCounter another) {
 getBitmap().and(another.getBitmap());
-count = getBitmap().getCardinality();
+count = null;
 }
 
 public int getCount() {
@@ -148,7 +150,6 @@ public class BitmapCounter implements 
Comparable {
 
 // keep forward compatibility
 if (version == VERSION) {
-@SuppressWarnings("unused")
 int size = in.getInt();
 count = in.getInt();
 in.position(mark);
@@ -258,4 +259,5 @@ public class BitmapCounter implements 
Comparable {
 return len;
 }
 }
+
 }

[31/50] [abbrv] kylin git commit: KYLIN-2331 by layer spark cubing

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
index 8951b69..710f324 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
@@ -84,6 +84,24 @@ public class MeasureAggregators implements Serializable {
 }
 }
 
+public void aggregate(Object[] values1, Object[] values2, Object[] result) 
{
+assert values1.length == values2.length && values2.length == 
descLength && values1.length == result.length;
+
+for (int i = 0; i < descLength; i++) {
+result[i] = aggs[i].aggregate(values1[i], values2[i]);
+}
+
+}
+
+public void aggregate(Object[] values1, Object[] values2, Object[] result, 
boolean[] aggrMask) {
+assert values1.length == values2.length && values2.length == 
descLength && values1.length == result.length && result.length == 
aggrMask.length;
+for (int i = 0; i < descLength; i++) {
+if (aggrMask[i]) {
+result[i] = aggs[i].aggregate(values1[i], values2[i]);
+}
+}
+}
+
 public void collectStates(Object[] states) {
 for (int i = 0; i < descLength; i++) {
 states[i] = aggs[i].getState();

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
index edaf806..2d73e59 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
@@ -18,19 +18,19 @@
 
 package org.apache.kylin.measure;
 
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 import org.apache.kylin.metadata.model.MeasureDesc;
 
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
 /**
  * @author yangli9
  * 
  */
 @SuppressWarnings({ "rawtypes" })
-public class MeasureCodec {
+public class MeasureCodec implements java.io.Serializable {
 
 private int nMeasures;
 private DataTypeSerializer[] serializers;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
index 0076252..26b7298 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
@@ -18,14 +18,14 @@
 
 package org.apache.kylin.measure;
 
-import java.util.Collection;
-import java.util.Map;
-
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 
-abstract public class MeasureIngester {
+import java.util.Collection;
+import java.util.Map;
+
+abstract public class MeasureIngester implements java.io.Serializable {
 
 public static MeasureIngester create(MeasureDesc measure) {
 return measure.getFunction().getMeasureType().newIngester();
@@ -42,6 +42,10 @@ abstract public class MeasureIngester {
 
 abstract public V valueOf(String[] values, MeasureDesc measureDesc, 
Map dictionaryMap);
 
+public void reset() {
+
+}
+
 public V reEncodeDictionary(V value, MeasureDesc measureDesc, 
Map oldDicts, Map 
newDicts) {
 throw new UnsupportedOperationException();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 89ff382..3338c8c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -18,11 +18,6 @@
 
 package org.apache.kylin.measure;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;

[50/50] [abbrv] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[lidong]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/094b9451
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/094b9451
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/094b9451

Branch: refs/heads/master-cdh5.7
Commit: 094b94510a938e1cd853b5caa2fcf09aaa504f05
Parents: c47f4ad
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Thu Jan 12 12:05:30 2017 +0800

--
 .../kylin/engine/mr/steps/MockupMapContext.java |  13 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  29 +-
 tool/pom.xml|  12 +
 10 files changed, 430 insertions(+), 1137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/094b9451/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index dfbba14..bc9601f 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -44,10 +44,10 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
 
 /**
  * @author yangli9
- * 
+ *
  */
 @SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext implements MapContext{
+public class MockupMapContext implements MapContext {
 
 private Configuration hconf;
 
@@ -60,7 +60,7 @@ public class MockupMapContext implements MapContext{
 return new WrappedMapper().getMapContext(new MockupMapContext(hconf, 
outKV));
 }
 
-public MockupMapContext(Configuration hconf, Object[] outKV){
+public MockupMapContext(Configuration hconf, Object[] outKV) {
 this.hconf = hconf;
 this.outKV = outKV;
 }
@@ -175,6 +175,11 @@ public class MockupMapContext implements MapContext{
 }
 
 @Override
+public boolean userClassesTakesPrecedence() {
+throw new NotImplementedException();
+}
+
+@Override
 public Class> getInputFormatClass() throws 
ClassNotFoundException {
 throw new NotImplementedException();
 }
@@ -318,4 +323,4 @@ public class MockupMapContext implements MapContext{
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/094b9451/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+
org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec
 
-
 
-hadoop.http.authentication.simple.anonymous.allowed
-true
+hadoop.security.authentication
+simple
 
-
 
-hadoop.proxyuser.falcon.groups
-users
+hadoop.security.authorization
+false
+
+
+hadoop.rpc.protection
+authentication
+
+
+hadoop.security.auth_to_local
+DEFAULT
 
-
 
-hadoop.proxyuser.falcon.hosts
+hadoop.proxyuser.oozie.hosts
 *
 
-
 
-hadoop.proxyuser.hbase.groups
-users
+

[43/50] [abbrv] kylin git commit: KYLIN-2380 refactor DbUnit assertion

[lidong]

KYLIN-2380 refactor DbUnit assertion


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f6450d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f6450d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f6450d6

Branch: refs/heads/master-cdh5.7
Commit: 5f6450d6e77f9f62c3670c20ec4d8aef18bfd2fb
Parents: 78d7086
Author: Li Yang 
Authored: Wed Jan 11 12:10:36 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 12:39:24 2017 +0800

--
 .../apache/kylin/query/HackedDbUnitAssert.java  | 134 ---
 .../apache/kylin/query/ITMassInQueryTest.java   |   9 +-
 .../org/apache/kylin/query/KylinTestBase.java   |  65 +
 3 files changed, 152 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f6450d6/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index c295430..3a21570 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -26,16 +26,34 @@ import org.dbunit.dataset.Columns;
 import org.dbunit.dataset.DataSetException;
 import org.dbunit.dataset.ITable;
 import org.dbunit.dataset.ITableMetaData;
+import org.dbunit.dataset.datatype.BigIntegerDataType;
 import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.IntegerDataType;
+import org.dbunit.dataset.datatype.UnknownDataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * dirty hack to support checking result of SQL with limit
+ * A copy of DbUnitAssert, in which a few hacks applied
+ * 
+ * - tolerate some column type difference, like INT vs. BIGINT
+ * - check expected table contains actual table (instead of equals), for sql 
with limit
  */
 public class HackedDbUnitAssert extends DbUnitAssert {
 private static final Logger logger = 
LoggerFactory.getLogger(HackedDbUnitAssert.class);
 
+private boolean hackCheckContains;
+private boolean hackIgnoreIntBigIntMismatch;
+
+public void hackCheckContains() {
+hackCheckContains = true;
+}
+
+public void hackIgnoreIntBigIntMismatch() {
+hackIgnoreIntBigIntMismatch = true;
+}
+
+// THIS METHOD IS MOSTLY COPIED FROM DbUnitAssert. CHANGES ARE LEAD BY 
hackXXX CONDITION CHECKS.
 public void assertEquals(ITable expectedTable, ITable actualTable, 
FailureHandler failureHandler) throws DatabaseUnitException {
 logger.trace("assertEquals(expectedTable, actualTable, failureHandler) 
- start");
 logger.debug("assertEquals: expectedTable={}", expectedTable);
@@ -57,25 +75,21 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 ITableMetaData actualMetaData = actualTable.getTableMetaData();
 String expectedTableName = expectedMetaData.getTableName();
 
-//// Verify row count
-//int expectedRowsCount = expectedTable.getRowCount();
-//int actualRowsCount = actualTable.getRowCount();
-//if (expectedRowsCount != actualRowsCount) {
-//String msg = "row count (table=" + expectedTableName + 
")";
-//Error error =
-//failureHandler.createFailure(msg, String
-//.valueOf(expectedRowsCount), String
-//.valueOf(actualRowsCount));
-//logger.error(error.toString());
-//throw error;
-//}
-
-// if both tables are empty, it is not necessary to compare columns, as
-// such
-// comparison
-// can fail if column metadata is different (which could occurs when
-// comparing empty tables)
-if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
+// Verify row count
+int expectedRowsCount = expectedTable.getRowCount();
+int actualRowsCount = actualTable.getRowCount();
+if (!hackCheckContains) {
+if (expectedRowsCount != actualRowsCount) {
+String msg = "row count (table=" + expectedTableName + ")";
+Error error = failureHandler.createFailure(msg, 
String.valueOf(expectedRowsCount), String.valueOf(actualRowsCount));
+logger.error(error.toString());
+throw error;
+}
+}
+
+// if both tables are empty, it is not necessary to compare columns, 
as such comparison
+// can fail if column metadata is different 

[02/50] [abbrv] kylin git commit: KYLIN-2358 CuboidReducer has too many if (aggrMask[i]) checks

[lidong]

KYLIN-2358 CuboidReducer has too many if (aggrMask[i]) checks


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b5257a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b5257a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b5257a1

Branch: refs/heads/master-cdh5.7
Commit: 4b5257a1dc4b1cf90804e5273f27944de30b8fbf
Parents: 9c98574
Author: shaofengshi 
Authored: Thu Jan 5 13:49:08 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 5 13:49:08 2017 +0800

--
 .../kylin/measure/MeasureAggregators.java   |  9 +++
 .../kylin/engine/mr/steps/CuboidReducer.java| 26 
 2 files changed, 19 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4b5257a1/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
index 2b28302..8951b69 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
@@ -77,13 +77,10 @@ public class MeasureAggregators implements Serializable {
 }
 }
 
-public void aggregate(Object[] values, boolean[] aggrMask) {
+public void aggregate(Object[] values, int[] measures) {
 assert values.length == descLength;
-assert aggrMask.length == descLength;
-
-for (int i = 0; i < descLength; i++) {
-if (aggrMask[i])
-aggs[i].aggregate(values[i]);
+for (int i = 0; i < measures.length; i++) {
+aggs[measures[i]].aggregate(values[measures[i]]);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/4b5257a1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index afd29e3..495be77 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import com.google.common.collect.Lists;
 import org.apache.hadoop.io.Text;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.CubeManager;
@@ -51,7 +52,7 @@ public class CuboidReducer extends KylinReducer {
 private MeasureAggregators aggs;
 
 private int cuboidLevel;
-private boolean[] needAggr;
+private int[] needAggrMeasures;
 private Object[] input;
 private Object[] result;
 private int vcounter;
@@ -76,13 +77,22 @@ public class CuboidReducer extends KylinReducer {
 
 input = new Object[measuresDescs.size()];
 result = new Object[measuresDescs.size()];
-needAggr = new boolean[measuresDescs.size()];
 
-if (cuboidLevel > 0) {
-for (int i = 0; i < measuresDescs.size(); i++) {
-needAggr[i] = 
!measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid();
+List needAggMeasuresList = Lists.newArrayList();
+for (int i = 0; i < measuresDescs.size(); i++) {
+if (cuboidLevel == 0) {
+needAggMeasuresList.add(i);
+} else {
+if 
(!measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid()) {
+needAggMeasuresList.add(i);
+}
 }
 }
+
+needAggrMeasures = new int[needAggMeasuresList.size()];
+for (int i = 0; i < needAggMeasuresList.size(); i++) {
+needAggrMeasures[i] = needAggMeasuresList.get(i);
+}
 }
 
 @Override
@@ -94,11 +104,7 @@ public class CuboidReducer extends KylinReducer {
 logger.info("Handling value with ordinal (This is not KV 
number!): " + vcounter);
 }
 codec.decode(ByteBuffer.wrap(value.getBytes(), 0, 
value.getLength()), input);
-if (cuboidLevel > 0) {
-aggs.aggregate(input, needAggr);
-} else {
-aggs.aggregate(input);
-}
+aggs.aggregate(input, needAggrMeasures);
 }
 aggs.collectStates(result);
 

[33/50] [abbrv] kylin git commit: KYLIN-2344 Package spark into Kylin binary package

[lidong]

KYLIN-2344 Package spark into Kylin binary package


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e864cd3b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e864cd3b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e864cd3b

Branch: refs/heads/master-cdh5.7
Commit: e864cd3b1c06700b1a1054d1f520eaabedc25d82
Parents: b7d87bb
Author: shaofengshi 
Authored: Tue Jan 3 10:28:38 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 build/bin/check-env.sh  |  6 ++
 build/bin/find-spark-dependency.sh  | 45 +++
 build/bin/kylin.sh  |  6 ++
 build/conf/kylin-spark-conf.properties  |  2 +-
 build/conf/kylin.properties |  3 -
 build/script/compress.sh|  5 +-
 build/script/download-spark.sh  | 52 +
 build/script/functions.sh   | 60 
 build/script/package.sh |  1 +
 .../org/apache/kylin/common/KylinConfig.java| 14 +
 .../apache/kylin/common/KylinConfigBase.java| 27 -
 .../spark/SparkBatchCubingJobBuilder2.java  |  4 +-
 12 files changed, 200 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/bin/check-env.sh
--
diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh
index a4003c9..e446d66 100644
--- a/build/bin/check-env.sh
+++ b/build/bin/check-env.sh
@@ -47,3 +47,9 @@ if [ $? != 0 ]
 then
 quit "Failed to create $WORKING_DIR. Please make sure the user has right 
to access $WORKING_DIR"
 fi
+
+hadoop fs -mkdir -p $WORKING_DIR/spark-history
+if [ $? != 0 ]
+then
+quit "Failed to create $WORKING_DIR/spark-history. Please make sure the 
user has right to access $WORKING_DIR"
+fi
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/bin/find-spark-dependency.sh
--
diff --git a/build/bin/find-spark-dependency.sh 
b/build/bin/find-spark-dependency.sh
new file mode 100644
index 000..6f74d8a
--- /dev/null
+++ b/build/bin/find-spark-dependency.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
+spark_home=
+
+verbose Retrieving Spark dependency...
+if [ -n "$SPARK_HOME" ]
+then
+verbose "SPARK_HOME is set to: $SPARK_HOME, use it to locate Spark 
dependencies."
+spark_home=$SPARK_HOME
+fi
+
+if [ -z "$SPARK_HOME" ]
+then
+verbose "SPARK_HOME wasn't set, use $KYLIN_HOME/spark"
+spark_home=$KYLIN_HOME/spark
+fi
+
+spark_dependency=`find -L $spark_home -name 
'spark-assembly-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name 
'*sources*' ''-printf '%p:' | sed 's/:$//'`
+if [ -z "$spark_dependency" ]
+then
+quit "spark assembly lib not found"
+else
+verbose "spark dependency: $spark_dependency"
+export spark_dependency
+fi
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 0cdbbc6..7813b79 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -45,6 +45,11 @@ function retrieveDependency() {
 source ${dir}/find-kafka-dependency.sh
 export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
 fi
+
+source ${dir}/find-spark-dependency.sh
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${spark_dependency}
+
+verbose "HBASE_CLASSPATH: ${HBASE_CLASSPATH}"
 }
 
 # start command
@@ -112,6 +117,7 @@ then
 -Dkylin.hive.dependency=${hive_dependency} \
 -Dkylin.hbase.dependency=${hbase_dependency} \
 -Dkylin.kafka.dependency=${kafka_dependency} \
+-Dkylin.spark.dependency=${spark_dependency} \
  

[39/50] [abbrv] kylin git commit: KYLIN-2367 ammend test case

[lidong]

KYLIN-2367 ammend test case


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07e5e5a7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07e5e5a7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07e5e5a7

Branch: refs/heads/master-cdh5.7
Commit: 07e5e5a75f4f692a53ac48c64c3cb44b84022c88
Parents: b0aadb1
Author: Li Yang 
Authored: Tue Jan 10 15:45:05 2017 +0800
Committer: Li Yang 
Committed: Tue Jan 10 15:45:05 2017 +0800

--
 .../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/07e5e5a7/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 2b33e66..df53de9 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -48,7 +48,7 @@
   }, {
 "id" : "10",
 "name" : "ITEM_COUNT",
-"datatype" : "int",
+"datatype" : "bigint",
 "data_gen" : "RAND"
   }, {
 "id" : "11",

[24/50] [abbrv] kylin git commit: KYLIN-2364 Output table name to error info in LookupTable

[lidong]

KYLIN-2364 Output table name to error info in LookupTable

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e46d699e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e46d699e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e46d699e

Branch: refs/heads/master-cdh5.7
Commit: e46d699e05100db084db354d7efb3786575d5c54
Parents: 56daf6d
Author: kangkaisen 
Authored: Sat Jan 7 19:43:43 2017 +0800
Committer: Billy Liu 
Committed: Sun Jan 8 12:14:05 2017 +0800

--
 .../src/main/java/org/apache/kylin/dict/lookup/LookupTable.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e46d699e/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index 21221f4..90ca500 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -83,7 +83,7 @@ abstract public class LookupTable {
 Array key = new Array(keyCols);
 
 if (data.containsKey(key))
-throw new IllegalStateException("Dup key found, key=" + 
toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + 
toString(value));
+throw new IllegalStateException("The table: " + 
tableDesc.getName() + " Dup key found, key=" + toString(keyCols) + ", value1=" 
+ toString(data.get(key)) + ", value2=" + toString(value));
 
 data.put(key, value);
 }

[05/50] [abbrv] kylin git commit: KYLIN-2333 fix ArrayIndexOutOfBoundsException in JobBuilderSupport

[lidong]

KYLIN-2333 fix ArrayIndexOutOfBoundsException in JobBuilderSupport


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5683954b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5683954b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5683954b

Branch: refs/heads/master-cdh5.7
Commit: 5683954bafec3fb02cc2ccda56387dacceb6b593
Parents: b736175
Author: shaofengshi 
Authored: Thu Jan 5 16:32:28 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 5 16:32:28 2017 +0800

--
 .../main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5683954b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 122b8ba..86ac880 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -193,7 +193,7 @@ public class JobBuilderSupport {
 }
 
 public String[] getCuboidOutputPaths(String cuboidRootPath, int levels) {
-String[] paths = new String[levels];
+String[] paths = new String[levels + 1];
 for (int i = 0; i <= levels; i++) {
 if (i == 0) {
 paths[i] = cuboidRootPath + "base_cuboid";

[34/50] [abbrv] kylin git commit: KYLIN-2331 use org.apache.htrace.Trace to find htrace-core.jar

[lidong]

KYLIN-2331 use org.apache.htrace.Trace to find htrace-core.jar


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b9a69efa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b9a69efa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b9a69efa

Branch: refs/heads/master-cdh5.7
Commit: b9a69efaea27d915ae44f75db436b6c9b6b7a4ec
Parents: e864cd3
Author: shaofengshi 
Authored: Tue Jan 3 13:36:49 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 .../apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a69efa/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index c5d47e7..9431468 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -59,10 +59,12 @@ public class SparkBatchCubingJobBuilder2 extends 
BatchCubingJobBuilder2 {
 StringBuilder jars = new StringBuilder();
 
 StringUtil.appendWithSeparator(jars, 
findJar("org.htrace.HTraceConfiguration")); // htrace-core.jar
-StringUtil.appendWithSeparator(jars, 
findJar("org.cloudera.htrace.HTraceConfiguration"));
+StringUtil.appendWithSeparator(jars, 
findJar("org.apache.htrace.Trace")); // htrace-core.jar
+StringUtil.appendWithSeparator(jars, 
findJar("org.cloudera.htrace.HTraceConfiguration")); // htrace-core.jar
 StringUtil.appendWithSeparator(jars, 
findJar("org.apache.hadoop.hbase.client.HConnection")); // hbase-client.jar
 StringUtil.appendWithSeparator(jars, 
findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar
 StringUtil.appendWithSeparator(jars, 
findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar
+StringUtil.appendWithSeparator(jars, 
findJar("com.yammer.metrics.core.Gauge")); // metrics-core.jar
 
 StringUtil.appendWithSeparator(jars, 
seg.getConfig().getSparkAdditionalJars());
 sparkExecutable.setJars(jars.toString());

[26/50] [abbrv] kylin git commit: KYLIN-2367 fix select * where ...

[lidong]

KYLIN-2367 fix select * where ...


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f8cdeb5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f8cdeb5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f8cdeb5

Branch: refs/heads/master-cdh5.7
Commit: 6f8cdeb55cb4b41a63871aed2b3f61c7eddd107d
Parents: 68ff75d
Author: Li Yang 
Authored: Mon Jan 9 14:41:05 2017 +0800
Committer: Li Yang 
Committed: Mon Jan 9 14:41:05 2017 +0800

--
 .../org/apache/calcite/sql2rel/SqlToRelConverter.java | 14 +-
 1 file changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6f8cdeb5/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 7da3040..e0782ce 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -677,7 +677,8 @@ public class SqlToRelConverter {
  * 
  *   LogicalSort (optional)
  *|- LogicalProject
- *|- OLAPTableScan
+ *|- LogicalFilter (optional)
+ *|- OLAPTableScan
  */
 LogicalProject rootPrj = null;
 LogicalSort rootSort = null;
@@ -690,11 +691,14 @@ public class SqlToRelConverter {
 return root;
 }
 
-if 
(!rootPrj.getInput().getClass().getSimpleName().equals("OLAPTableScan"))
+RelNode input = rootPrj.getInput();
+if (!(//
+input.getClass().getSimpleName().equals("OLAPTableScan")//
+|| (input.getClass().getSimpleName().equals("LogicalFilter") 
&& input.getInput(0).getClass().getSimpleName().equals("OLAPTableScan"))//
+ ))
 return root;
 
-RelNode scan = rootPrj.getInput();
-if (rootPrj.getRowType().getFieldCount() < 
scan.getRowType().getFieldCount())
+if (rootPrj.getRowType().getFieldCount() < 
input.getRowType().getFieldCount())
 return root;
 
 RelDataType inType = rootPrj.getRowType();
@@ -713,7 +717,7 @@ public class SqlToRelConverter {
 }
 
 RelDataType projRowType = 
getCluster().getTypeFactory().createStructType(projTypeBuilder);
-rootPrj = LogicalProject.create(scan, projExp, projRowType);
+rootPrj = LogicalProject.create(input, projExp, projRowType);
 if (rootSort != null) {
 rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), 
rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch);
 }

[21/50] [abbrv] kylin git commit: KYLIN-2349 Serialize BitmapCounter with peekLength

[lidong]

KYLIN-2349 Serialize BitmapCounter with peekLength

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa574462
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa574462
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa574462

Branch: refs/heads/master-cdh5.7
Commit: aa574462aad0620fe10343613dc4c8d761c0d6a0
Parents: 746e380
Author: kangkaisen 
Authored: Sat Dec 31 15:41:07 2016 +0800
Committer: Yang Li 
Committed: Sat Jan 7 19:08:15 2017 +0800

--
 .../kylin/measure/bitmap/BitmapCounter.java | 53 ++--
 1 file changed, 37 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aa574462/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index a7f277e..aeb14ba 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -34,6 +34,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
 public class BitmapCounter implements Comparable {
 
 private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+private final int VERSION = 2;
 
 public BitmapCounter() {
 }
@@ -102,15 +103,51 @@ public class BitmapCounter implements 
Comparable {
 bitmap.serialize(dos);
 dos.close();
 ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray());
+out.putInt(VERSION);
+out.putInt(bos.size() + 4 + 4);
 out.put(bb);
 }
 
 public void readRegisters(ByteBuffer in) throws IOException {
+int mark = in.position();
+int version = in.getInt();
+
+// keep forward compatibility
+if (version == VERSION) {
+@SuppressWarnings("unused")
+int size = in.getInt();
+} else {
+in.position(mark);
+}
+
 try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(in))) {
 bitmap.deserialize(is);
 }
 }
 
+public int peekLength(ByteBuffer in) {
+int mark = in.position();
+int len;
+int version = in.getInt();
+
+// keep forward compatibility
+if (version == VERSION) {
+len = in.getInt() ;
+} else {
+in.position(mark);
+try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(in))) {
+MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+bitmap.deserialize(is);
+len = in.position() - mark;
+} catch (IOException e) {
+throw new IllegalStateException(e);
+}
+}
+
+in.position(mark);
+return len;
+}
+
 @Override
 public String toString() {
 long count = getCount();
@@ -169,22 +206,6 @@ public class BitmapCounter implements 
Comparable {
 return -1;
 }
 
-public int peekLength(ByteBuffer in) {
-int mark = in.position();
-int len;
-
-MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
-try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(in))) {
-bitmap.deserialize(is);
-} catch (IOException e) {
-throw new IllegalStateException(e);
-}
-
-len = in.position() - mark;
-in.position(mark);
-return len;
-}
-
 private class ByteBufferBackedInputStream extends InputStream {
 private final ByteBuffer buffer;
 

[07/50] [abbrv] kylin git commit: KYLIN-2356 Incorrect result when filter on numeric columns

[lidong]

KYLIN-2356 Incorrect result when filter on numeric columns

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/980b03b1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/980b03b1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/980b03b1

Branch: refs/heads/master-cdh5.7
Commit: 980b03b1abb8af82a9bceeb497847fe17c19c3e3
Parents: c64f55b
Author: zhengdong 
Authored: Thu Jan 5 15:05:35 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 5 17:45:20 2017 +0800

--
 .../filter/FilterCodeSystemFactory.java | 98 
 .../translate/DerivedFilterTranslator.java  |  4 +-
 2 files changed, 100 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/980b03b1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
new file mode 100755
index 000..bae8cf9
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+
+/**
+ * Created by donald.zheng on 2016/12/19.
+ */
+public class FilterCodeSystemFactory {
+private final static HashMap codeSystemMap = 
new HashMap<>();
+
+static {
+codeSystemMap.put("string", StringCodeSystem.INSTANCE);
+codeSystemMap.put("integer", new IntegerCodeSystem());
+codeSystemMap.put("decimal", new DecimalCodeSystem());
+}
+
+public static IFilterCodeSystem getFilterCodeSystem(DataType dataType) {
+if (dataType.isIntegerFamily()) {
+return codeSystemMap.get("integer");
+} else if (dataType.isNumberFamily()) {
+return codeSystemMap.get("decimal");
+} else {
+return codeSystemMap.get("string");
+}
+}
+
+private static class IntegerCodeSystem implements IFilterCodeSystem {
+
+@Override
+public boolean isNull(Object code) {
+return code == null;
+}
+
+@Override
+public void serialize(Object code, ByteBuffer buf) {
+BytesUtil.writeLong(Long.parseLong(code.toString()), buf);
+}
+
+@Override
+public Object deserialize(ByteBuffer buf) {
+return BytesUtil.readLong(buf);
+}
+
+@Override
+public int compare(Object o, Object t1) {
+long l1 = Long.parseLong(o.toString());
+long l2 = Long.parseLong(t1.toString());
+return Long.compare(l1, l2);
+}
+}
+
+private static class DecimalCodeSystem implements IFilterCodeSystem {
+@Override
+public boolean isNull(Object code) {
+return code == null;
+}
+
+@Override
+public void serialize(Object code, ByteBuffer buf) {
+BytesUtil.writeUTFString(code.toString(), buf);
+}
+
+@Override
+public Object deserialize(ByteBuffer buf) {
+return Double.parseDouble(BytesUtil.readUTFString(buf));
+}
+
+@Override
+public int compare(Object o, Object t1) {
+double d1 = Double.parseDouble(o.toString());
+double d2 = Double.parseDouble(t1.toString());
+return Double.compare(d1, d2);
+}
+}
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/980b03b1/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java

[29/50] [abbrv] kylin git commit: KYLIN-2371 Allow overwrite default spark conf at cube level

[lidong]

KYLIN-2371 Allow overwrite default spark conf at cube level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ee74a74e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ee74a74e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ee74a74e

Branch: refs/heads/master-cdh5.7
Commit: ee74a74e4d17a45b3832912381220af39e71f3d5
Parents: 66bca9a
Author: shaofengshi 
Authored: Mon Jan 9 15:23:22 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 build/conf/kylin-spark-conf.properties  |  5 +--
 build/conf/kylin.properties |  6 ++--
 .../apache/kylin/common/KylinConfigBase.java| 12 ---
 .../kylin/job/constant/ExecutableConstants.java |  1 +
 .../spark/SparkBatchCubingJobBuilder2.java  |  2 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |  7 ++--
 .../kylin/engine/spark/SparkExecutable.java | 38 +---
 .../test_case_data/sandbox/kylin.properties |  3 +-
 8 files changed, 55 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/build/conf/kylin-spark-conf.properties
--
diff --git a/build/conf/kylin-spark-conf.properties 
b/build/conf/kylin-spark-conf.properties
index 5e6dafe..143e7e4 100644
--- a/build/conf/kylin-spark-conf.properties
+++ b/build/conf/kylin-spark-conf.properties
@@ -1,5 +1,5 @@
 spark.yarn.submit.file.replication=1
-spark.yarn.executor.memoryOverhead=200
+spark.yarn.executor.memoryOverhead=1024
 spark.yarn.driver.memoryOverhead=384
 spark.master=yarn
 spark.submit.deployMode=cluster
@@ -20,7 +20,8 @@ spark.executor.cores=4
 spark.executor.instances=8
 spark.history.kerberos.keytab=none
 spark.history.kerberos.principal=none
-#spark.yarn.jar=hdfs://namenode:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 spark.driver.extraJavaOptions=-Dhdp.version=current
 spark.yarn.am.extraJavaOptions=-Dhdp.version=current
 spark.executor.extraJavaOptions=-Dhdp.version=current
+#spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+#spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec

http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 54430f0..6efa423 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -131,16 +131,16 @@ kylin.engine.mr.mapper-input-rows=100
 ### Spark Engine ###
 
 # Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run 
spark-submit
-kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+#kylin.engine.spark.env.hadoop-conf-dir=/etc/hive/conf
 
 # Spark job submission properties file, default be 
$KYLIN_HOME/conf/kylin-spark-conf.properties
 #kylin.engine.spark.properties-file=
 
 # Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=50
+kylin.engine.spark.rdd-partition-cut-mb=10
 
 # Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=10
+kylin.engine.spark.min-partition=1
 
 # Max partition numbers of rdd
 kylin.engine.spark.max-partition=5000

http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7d6ac2b..36ddbf4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -695,6 +695,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.engine.mr.config-override.");
 }
 
+public Map getSparkConfigOverride() {
+return getPropertiesByPrefix("kylin.engine.spark.config-override.");
+}
+
 public double getDefaultHadoopJobReducerInputMB() {
 return 
Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500"));
 }
@@ -739,8 +743,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 // ENGINE.SPARK
 // 

 
-public String getSparkHadoopConfDir() {
-return getRequired("kylin.engine.spark.env.hadoop-conf-dir");
+public String getHadoopConfDir() {
+return getOptional("kylin.engine.spark.env.hadoop-conf-dir", "");
 }
 
 public String getSparkConfFile() {
@@ -764,7 

[12/50] [abbrv] kylin git commit: KYLIN-2287 fix the button display issue with js

[lidong]

KYLIN-2287 fix the button display issue with js

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49de556f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49de556f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49de556f

Branch: refs/heads/master-cdh5.7
Commit: 49de556f00ff8fd854654daf1bc7e1d26c1fb879
Parents: f267ba9
Author: ShaoChinYu 
Authored: Fri Jan 6 12:22:59 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 6 12:29:35 2017 +0800

--
 webapp/app/js/controllers/models.js | 36 
 webapp/app/partials/models/models_tree.html |  6 ++--
 2 files changed, 39 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/49de556f/webapp/app/js/controllers/models.js
--
diff --git a/webapp/app/js/controllers/models.js 
b/webapp/app/js/controllers/models.js
index e3ee038..fd21845 100644
--- a/webapp/app/js/controllers/models.js
+++ b/webapp/app/js/controllers/models.js
@@ -183,7 +183,41 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, 
$routeParams, $location,
 });
   };
 
+  function changePositionOfScrollBar(){
+//get which button be clicked
+var btn = window.event.srcElement || window.event.target;
+//get current position of scroll bar
+var scrollTop =$("#cube_model_trees").scrollTop();
+//get total length of scroll bar
+var scrollHeight  = 
document.getElementById('cube_model_trees').scrollHeight;
+//get the position of clicked button relative to the top of window
+var offsetTop  =$(btn).offset().top;
+//get the position of the container relative to the top of window
+var treeOffsetTop = $("#cube_model_trees").offset().top;
+
+//distance from button to the top of tree model container
+var minor = offsetTop - treeOffsetTop;
+//height of tree model container
+var  viewH =$("#cube_model_trees").height();
+
+//change scroll bar to show the dropdown menu
+if(minor + 100 > viewH){//100 is the height of dropdowm menu
+  if((scrollHeight - scrollTop - viewH)>=minor+100-viewH){
+
document.getElementById('cube_model_trees').scrollTop+=(minor+120-viewH);
+  }else{
+var node=document.createElement("LI");
+node.style.height = (minor+120-viewH)+"px";
+document.getElementById("models-tree").appendChild(node);
+var  viewH =$("#cube_model_trees").height();//可见高度
+
document.getElementById('cube_model_trees').scrollTop+=(minor+120-viewH);
+
+  }
+}
+  }
+
   $scope.listModelAccess = function (model) {
+changePositionOfScrollBar();
+
 if(model.uuid){
   AccessService.list({type: "DataModelDesc", uuid: model.uuid}, function 
(accessEntities) {
 model.accessEntities = accessEntities;
@@ -196,6 +230,8 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, 
$routeParams, $location,
 }
   })
 }
+
+
   };
 
   var ModelDetailModalCtrl = function ($scope, $location, $modalInstance, 
scope) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/49de556f/webapp/app/partials/models/models_tree.html
--
diff --git a/webapp/app/partials/models/models_tree.html 
b/webapp/app/partials/models/models_tree.html
index 944d2dc..eba7d29 100644
--- a/webapp/app/partials/models/models_tree.html
+++ b/webapp/app/partials/models/models_tree.html
@@ -41,10 +41,10 @@
   
 Models
   
+
+
 
-
-
-
+
   
 
 

[36/50] [abbrv] kylin git commit: KYLIN-2371 rename the prefix to ‘kylin.engine.spark-conf.’

[lidong]

KYLIN-2371 rename the prefix to ‘kylin.engine.spark-conf.’


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00c8f31e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00c8f31e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00c8f31e

Branch: refs/heads/master-cdh5.7
Commit: 00c8f31e6d235d38bb7919fa4ee5b12e53d840f6
Parents: ee74a74
Author: shaofengshi 
Authored: Tue Jan 10 09:40:44 2017 +0800
Committer: shaofengshi 
Committed: Tue Jan 10 09:40:44 2017 +0800

--
 build/conf/kylin-spark-conf.properties  | 27 ---
 build/conf/kylin.properties | 48 
 build/deploy/spark-defaults.conf|  6 +++
 build/script/download-spark.sh  |  3 +-
 .../apache/kylin/common/KylinConfigBase.java| 18 +---
 .../kylin/engine/mr/BatchCubingJobBuilder2.java |  2 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  | 10 ++--
 .../kylin/engine/spark/SparkExecutable.java |  7 ++-
 .../test_case_data/sandbox/kylin.properties | 19 ++--
 9 files changed, 65 insertions(+), 75 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/build/conf/kylin-spark-conf.properties
--
diff --git a/build/conf/kylin-spark-conf.properties 
b/build/conf/kylin-spark-conf.properties
deleted file mode 100644
index 143e7e4..000
--- a/build/conf/kylin-spark-conf.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-spark.yarn.submit.file.replication=1
-spark.yarn.executor.memoryOverhead=1024
-spark.yarn.driver.memoryOverhead=384
-spark.master=yarn
-spark.submit.deployMode=cluster
-spark.eventLog.enabled=true
-spark.yarn.scheduler.heartbeat.interval-ms=5000
-spark.yarn.preserve.staging.files=true
-spark.yarn.queue=default
-spark.yarn.containerLauncherMaxThreads=25
-spark.yarn.max.executor.failures=3
-spark.eventLog.dir=hdfs\:///kylin/spark-history
-spark.history.kerberos.enabled=true
-spark.history.provider=org.apache.spark.deploy.history.FsHistoryProvider
-spark.history.ui.port=18080
-spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
-spark.executor.memory=4G
-spark.storage.memoryFraction=0.3
-spark.executor.cores=4
-spark.executor.instances=8
-spark.history.kerberos.keytab=none
-spark.history.kerberos.principal=none
-spark.driver.extraJavaOptions=-Dhdp.version=current
-spark.yarn.am.extraJavaOptions=-Dhdp.version=current
-spark.executor.extraJavaOptions=-Dhdp.version=current
-#spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
-#spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec

http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 6efa423..196a711 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -127,24 +127,6 @@ kylin.engine.mr.max-reducer-number=500
 
 kylin.engine.mr.mapper-input-rows=100
 
-
-### Spark Engine ###
-
-# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run 
spark-submit
-#kylin.engine.spark.env.hadoop-conf-dir=/etc/hive/conf
-
-# Spark job submission properties file, default be 
$KYLIN_HOME/conf/kylin-spark-conf.properties
-#kylin.engine.spark.properties-file=
-
-# Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=10
-
-# Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=1
-
-# Max partition numbers of rdd
-kylin.engine.spark.max-partition=5000
-
 ### CUBE | DICTIONARY ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing
@@ -204,3 +186,33 @@ kylin.security.saml.context-server-name=hostname
 kylin.security.saml.context-server-port=443
 kylin.security.saml.context-path=/kylin
 
+
+### Spark Engine Configs ###
+
+# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run 
spark-submit
+#kylin.engine.spark.env.hadoop-conf-dir=/etc/hive/conf
+
+# Estimate the RDD partition numbers
+kylin.engine.spark.rdd-partition-cut-mb=10
+
+# Minimal partition numbers of rdd
+kylin.engine.spark.min-partition=1
+
+# Max partition numbers of rdd
+kylin.engine.spark.max-partition=5000
+
+### Spark conf (default is in spark/conf/spark-defaults.conf)
+kylin.engine.spark-conf.spark.master=yarn
+kylin.engine.spark-conf.spark.submit.deployMode=cluster
+kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=1024
+kylin.engine.spark-conf.spark.yarn.driver.memoryOverhead=384
+kylin.engine.spark-conf.spark.yarn.queue=default
+kylin.engine.spark-conf.spark.executor.memory=4G
+kylin.engine.spark-conf.spark.executor.cores=4

[30/50] [abbrv] kylin git commit: KYLIN-2331 by layer spark cubing

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
new file mode 100644
index 000..07b636b
--- /dev/null
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import com.google.common.collect.Sets;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ */
+public class BaseCuboidBuilder implements java.io.Serializable {
+
+protected static final Logger logger = 
LoggerFactory.getLogger(BaseCuboidBuilder.class);
+public static final String HIVE_NULL = "\\N";
+protected String cubeName;
+protected Cuboid baseCuboid;
+protected CubeDesc cubeDesc;
+protected CubeSegment cubeSegment;
+protected Set nullStrs;
+protected CubeJoinedFlatTableEnrich intermediateTableDesc;
+protected MeasureIngester[] aggrIngesters;
+protected Map dictionaryMap;
+protected AbstractRowKeyEncoder rowKeyEncoder;
+protected BufferedMeasureCodec measureCodec;
+
+protected KylinConfig kylinConfig;
+
+public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, 
CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc,
+ AbstractRowKeyEncoder rowKeyEncoder, 
MeasureIngester[] aggrIngesters, Map 
dictionaryMap) {
+this.kylinConfig = kylinConfig;
+this.cubeDesc = cubeDesc;
+this.cubeSegment = cubeSegment;
+this.intermediateTableDesc = intermediateTableDesc;
+this.rowKeyEncoder = rowKeyEncoder;
+this.aggrIngesters = aggrIngesters;
+this.dictionaryMap = dictionaryMap;
+
+init();
+measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+}
+
+public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, 
CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc) {
+this.kylinConfig = kylinConfig;
+this.cubeDesc = cubeDesc;
+this.cubeSegment = cubeSegment;
+this.intermediateTableDesc = intermediateTableDesc;
+
+init();
+rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, 
baseCuboid);
+measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+aggrIngesters = MeasureIngester.create(cubeDesc.getMeasures());
+dictionaryMap = cubeSegment.buildDictionaryMap();
+
+}
+
+private void init() {
+long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+initNullBytes();
+}
+
+private void initNullBytes() {
+nullStrs = Sets.newHashSet();
+nullStrs.add(HIVE_NULL);
+String[] nullStrings = cubeDesc.getNullStrings();
+if (nullStrings != null) {
+for (String s : nullStrings) {
+nullStrs.add(s);
+}
+}
+}
+
+protected boolean isNull(String v) {
+return nullStrs.contains(v);
+}
+
+public byte[] buildKey(String[] 

[42/50] [abbrv] kylin git commit: rollback ITEM_COUNT to INT type

[lidong]

rollback ITEM_COUNT to INT type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78d7086e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78d7086e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78d7086e

Branch: refs/heads/master-cdh5.7
Commit: 78d7086e51c9311ab25b17f05206766d8a279afa
Parents: 521dfff
Author: Li Yang 
Authored: Wed Jan 11 09:40:55 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 09:40:55 2017 +0800

--
 .../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/78d7086e/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index df53de9..2b33e66 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -48,7 +48,7 @@
   }, {
 "id" : "10",
 "name" : "ITEM_COUNT",
-"datatype" : "bigint",
+"datatype" : "int",
 "data_gen" : "RAND"
   }, {
 "id" : "11",

[20/50] [abbrv] kylin git commit: KYLIN-2338 refactor BitmapCounter.DataInputByteBuffer

[lidong]

KYLIN-2338 refactor BitmapCounter.DataInputByteBuffer

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/746e3801
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/746e3801
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/746e3801

Branch: refs/heads/master-cdh5.7
Commit: 746e3801faec58f0e10e619f0697ffbe37c08c63
Parents: 5a18af0
Author: kangkaisen 
Authored: Mon Dec 26 20:12:30 2016 +0800
Committer: Yang Li 
Committed: Sat Jan 7 19:04:52 2017 +0800

--
 .../kylin/measure/bitmap/BitmapCounter.java | 111 +++
 1 file changed, 18 insertions(+), 93 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/746e3801/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index 827390d..a7f277e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -26,7 +26,6 @@ import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 
-import org.apache.commons.io.IOUtils;
 import org.roaringbitmap.buffer.MutableRoaringBitmap;
 
 /**
@@ -107,12 +106,8 @@ public class BitmapCounter implements 
Comparable {
 }
 
 public void readRegisters(ByteBuffer in) throws IOException {
-DataInputByteBuffer input = new DataInputByteBuffer();
-try {
-input.reset(new ByteBuffer[] { in });
-bitmap.deserialize(input);
-} finally {
-IOUtils.closeQuietly(input);
+try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(in))) {
+bitmap.deserialize(is);
 }
 }
 
@@ -178,15 +173,11 @@ public class BitmapCounter implements 
Comparable {
 int mark = in.position();
 int len;
 
-DataInputByteBuffer input = new DataInputByteBuffer();
-input.reset(new ByteBuffer[] { in });
 MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
-try {
-bitmap.deserialize(input);
+try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(in))) {
+bitmap.deserialize(is);
 } catch (IOException e) {
 throw new IllegalStateException(e);
-} finally {
-IOUtils.closeQuietly(input);
 }
 
 len = in.position() - mark;
@@ -194,93 +185,27 @@ public class BitmapCounter implements 
Comparable {
 return len;
 }
 
-static class DataInputByteBuffer extends DataInputStream {
-private DataInputByteBuffer.Buffer buffers;
+private class ByteBufferBackedInputStream extends InputStream {
+private final ByteBuffer buffer;
 
-public DataInputByteBuffer() {
-this(new DataInputByteBuffer.Buffer());
+private ByteBufferBackedInputStream(ByteBuffer buf) {
+buffer = buf;
 }
 
-private DataInputByteBuffer(DataInputByteBuffer.Buffer buffers) {
-super(buffers);
-this.buffers = buffers;
+@Override
+public int read() throws IOException {
+return buffer.hasRemaining() ? (buffer.get() & 0xFF) : -1;
 }
 
-public void reset(ByteBuffer... input) {
-this.buffers.reset(input);
-}
-
-public ByteBuffer[] getData() {
-return this.buffers.getData();
-}
-
-public int getPosition() {
-return this.buffers.getPosition();
-}
-
-public int getLength() {
-return this.buffers.getLength();
-}
-
-private static class Buffer extends InputStream {
-private final byte[] scratch;
-ByteBuffer[] buffers;
-int bidx;
-int pos;
-int length;
-
-private Buffer() {
-this.scratch = new byte[1];
-this.buffers = new ByteBuffer[0];
-}
-
-public int read() {
-return -1 == this.read(this.scratch, 0, 1) ? -1 : 
this.scratch[0] & 255;
+@Override
+public int read(byte[] bytes, int off, int len) throws IOException {
+if (!buffer.hasRemaining()) {
+return -1;
 }
 
-public int read(byte[] b, int off, int len) {
-if (this.bidx >= this.buffers.length) {
-return -1;
-} else {
-int cur = 0;
-

[11/50] [abbrv] kylin git commit: KYLIN-2361 Upgrade to Tomcat 8.5.9

[lidong]

KYLIN-2361 Upgrade to Tomcat 8.5.9


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f267ba9d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f267ba9d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f267ba9d

Branch: refs/heads/master-cdh5.7
Commit: f267ba9dcb3db317d07eb367ddff668e03291519
Parents: f573486
Author: Billy Liu 
Authored: Fri Jan 6 10:42:48 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 6 10:42:48 2017 +0800

--
 build/deploy/context.xml| 47 ++---
 build/deploy/server.xml | 72 +---
 build/script/download-tomcat.sh | 14 ++--
 pom.xml |  2 +-
 .../java/org/apache/kylin/rest/DebugTomcat.java | 10 +--
 .../kylin/ext/CustomizedWebappClassloader.java  |  5 +-
 6 files changed, 82 insertions(+), 68 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/build/deploy/context.xml
--
diff --git a/build/deploy/context.xml b/build/deploy/context.xml
index 5d1bedf..e8b4cf1 100644
--- a/build/deploy/context.xml
+++ b/build/deploy/context.xml
@@ -1,38 +1,31 @@
-
+
 
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
 
 
 
-
+
+
 WEB-INF/web.xml
+${catalina.base}/conf/web.xml
 
 
 
-
-
-
-
 
-
-
\ No newline at end of file
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/build/deploy/server.xml
--
diff --git a/build/deploy/server.xml b/build/deploy/server.xml
index 72a87d0..bea6146 100644
--- a/build/deploy/server.xml
+++ b/build/deploy/server.xml
@@ -1,4 +1,4 @@
-
+
 
 
+
 
 
-
-
-
+
 
-
-
-
+
+
+
 
 
 
 -->
-
+
 
+
+
 
 
-
+
 
 
 
 
+   prefix="localhost_access_log" suffix=".txt"
+   pattern="%h %l %u %t %r %s %b" />
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/build/script/download-tomcat.sh
--
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index 9dd26f0..c97c3ce 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -27,22 +27,22 @@ if [[ `uname -a` =~ "Darwin" ]]; then
 alias md5cmd="md5 -q"
 fi
 
-if [ ! -f "build/apache-tomcat-7.0.69.tar.gz" ]
+if [ ! -f "build/apache-tomcat-8.5.9.tar.gz" ]
 then
 echo "no binary file found"
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.69/bin/apache-tomcat-7.0.69.tar.gz
 || echo "download tomcat failed"
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v8.5.9/bin/apache-tomcat-8.5.9.tar.gz
 || echo "download tomcat failed"
 else
-if [ `md5cmd build/apache-tomcat-7.0.69.tar.gz | awk '{print $1}'` != 
"10a071e5169a1a8b14ff35a0ad181052" ]
+if [ `md5cmd build/apache-tomcat-8.5.9.tar.gz | awk '{print $1}'` != 
"b41270a64b7774c964e4bec813eea2ed" ]
 then
 echo "md5 check failed"
-rm build/apache-tomcat-7.0.69.tar.gz
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.69/bin/apache-tomcat-7.0.69.tar.gz
 || echo "download tomcat failed"
+rm build/apache-tomcat-8.5.9.tar.gz
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v8.5.9/bin/apache-tomcat-8.5.9.tar.g
 || echo "download tomcat failed"
 fi
 fi
 unalias md5cmd
 
-tar -zxvf build/apache-tomcat-7.0.69.tar.gz -C build/
-mv build/apache-tomcat-7.0.69 build/tomcat
+tar -zxvf build/apache-tomcat-8.5.9.tar.gz -C build/
+mv build/apache-tomcat-8.5.9 build/tomcat
 rm -rf build/tomcat/webapps/*
 
 mv build/tomcat/conf/server.xml 

[01/50] [abbrv] kylin git commit: KYLIN-2359 Update job build step name [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master-cdh5.7 c5f4ac54b -> 094b94510 (forced update)


KYLIN-2359 Update job build step name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c985742
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c985742
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c985742

Branch: refs/heads/master-cdh5.7
Commit: 9c985742584a6bf15053d5482d13e1e01f76894d
Parents: 809b783
Author: shaofengshi 
Authored: Thu Jan 5 12:19:42 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 5 12:19:42 2017 +0800

--
 .../org/apache/kylin/job/constant/ExecutableConstants.java| 7 ---
 .../org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java| 4 ++--
 .../org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java| 4 ++--
 3 files changed, 8 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9c985742/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 4ae3add..11c7455 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -38,9 +38,9 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create 
Intermediate Flat Hive Table";
 public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = 
"Materialize Hive View in Lookup Tables";
 public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact 
Table Distinct Columns";
-public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base 
Cuboid Data";
-public static final String STEP_NAME_BUILD_IN_MEM_CUBE = "Build Cube";
-public static final String STEP_NAME_BUILD_N_D_CUBOID = "Build N-Dimension 
Cuboid Data";
+public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base 
Cuboid";
+public static final String STEP_NAME_BUILD_IN_MEM_CUBE = "Build Cube 
In-Mem";
+public static final String STEP_NAME_BUILD_N_D_CUBOID = "Build N-Dimension 
Cuboid";
 public static final String STEP_NAME_GET_CUBOID_KEY_DISTRIBUTION = 
"Calculate HTable Region Splits";
 public static final String STEP_NAME_CREATE_HBASE_TABLE = "Create HTable";
 public static final String STEP_NAME_CONVERT_CUBOID_TO_HFILE = "Convert 
Cuboid Data to HFile";
@@ -53,6 +53,7 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_HIVE_CLEANUP = "Hive Cleanup";
 public static final String STEP_NAME_KAFKA_CLEANUP = "Kafka Intermediate 
File Cleanup";
 public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage 
Collection";
+public static final String STEP_NAME_GARBAGE_COLLECTION_HBASE = "Garbage 
Collection on HBase";
 public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage 
Collection on HDFS";
 public static final String STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE = 
"Redistribute Flat Hive Table";
 public static final String NOTIFY_EMAIL_TEMPLATE = "Build Result 
of Job ${job_name}" + "Build Result: ${result}" + 
"Job Engine: ${job_engine}" + "Env: ${env_name}" + 
"Project: ${project_name}" + "Cube Name: ${cube_name}" + 
"Source Records Count: ${source_records_count}" + "Start Time: 
${start_time}" + "Duration: ${duration}" + "MR Waiting: 
${mr_waiting}" + "Last Update Time: ${last_update_time}" + 
"Submitter: ${submitter}" + "Error Log: ${error_log}" + 
"";

http://git-wip-us.apache.org/repos/asf/kylin/blob/9c985742/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 20540a6..9229a75 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -43,13 +43,13 @@ public class BatchCubingJobBuilder2 extends 
JobBuilderSupport {
 public BatchCubingJobBuilder2(CubeSegment newSegment, String submitter) {
 super(newSegment, submitter);
 this.inputSide = MRUtil.getBatchCubingInputSide(seg);
-this.outputSide = MRUtil.getBatchCubingOutputSide2((CubeSegment) seg);
+this.outputSide = MRUtil.getBatchCubingOutputSide2(seg);
 }
 
 public CubingJob build() 

[27/50] [abbrv] kylin git commit: KYLIN-2368 Enable Findbugs plugin

[lidong]

KYLIN-2368 Enable Findbugs plugin


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75625ae5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75625ae5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75625ae5

Branch: refs/heads/master-cdh5.7
Commit: 75625ae512b3d6cc080053e4e8871f3fff816ef0
Parents: 6f8cdeb
Author: Billy Liu 
Authored: Mon Jan 9 15:02:06 2017 +0800
Committer: Billy Liu 
Committed: Mon Jan 9 15:02:13 2017 +0800

--
 atopcalcite/pom.xml |  4 +--
 core-common/pom.xml |  5 +--
 core-cube/pom.xml   |  2 +-
 core-dictionary/pom.xml |  5 +--
 core-job/pom.xml|  5 +--
 core-metadata/pom.xml   |  5 +--
 core-storage/pom.xml|  5 +--
 engine-mr/pom.xml   |  6 +---
 engine-spark/pom.xml| 16 --
 jdbc/pom.xml|  3 +-
 kylin-it/pom.xml| 11 ---
 pom.xml | 73 +++-
 query/pom.xml   |  1 +
 server-base/pom.xml |  5 +--
 server/pom.xml  |  2 +-
 source-hive/pom.xml |  5 +--
 source-kafka/pom.xml|  2 +-
 storage-hbase/pom.xml   |  5 +--
 tomcat-ext/pom.xml  |  8 +++--
 tool/pom.xml| 11 ---
 20 files changed, 92 insertions(+), 87 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b1bc171..b916df2 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -24,6 +24,7 @@
 atopcalcite
 jar
 Apache Kylin - Calcite Overrides
+Apache Kylin - Calcite Overrides
 
 
 org.apache.kylin
@@ -31,9 +32,6 @@
 2.0.0-SNAPSHOT
 
 
-
-
-
 
 
 org.apache.calcite

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 2265651..25b10a7 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -24,17 +24,14 @@
 kylin-core-common
 jar
 Apache Kylin - Core Common
+Apache Kylin - Core Common
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
-
-
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index b7db363..0a30432 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -24,12 +24,12 @@
 kylin-core-cube
 jar
 Apache Kylin - Core Cube
+Apache Kylin - Core Cube
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index a74eccb..5d839dd 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -24,17 +24,14 @@
 kylin-core-dictionary
 jar
 Apache Kylin - Core Dictionary
+Apache Kylin - Core Dictionary
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
-
-
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index dad0428..36d34c8 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -24,17 +24,14 @@
 kylin-core-job
 jar
 Apache Kylin - Core Job
+Apache Kylin - Core Job
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
-
-
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index f5c8f8a..b3fe885 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -24,17 +24,14 @@
 kylin-core-metadata
 jar
 Apache Kylin - Core Metadata
+Apache Kylin - Core Metadata
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
-
-
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 8987608..9bd3f04 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -24,17 +24,14 @@
 kylin-core-storage
 jar
 Apache Kylin - Core 

[14/50] [abbrv] kylin git commit: KYLIN-2361, fix redundency dependency introduced by Tomcat 8

[lidong]

KYLIN-2361, fix redundency dependency introduced by Tomcat 8


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/40ede31b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/40ede31b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/40ede31b

Branch: refs/heads/master-hbase1.x
Commit: 40ede31bf538efb60c879914a6387f1eb107c5f1
Parents: 49de556
Author: Billy Liu 
Authored: Fri Jan 6 15:03:50 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 6 15:04:10 2017 +0800

--
 server/pom.xml | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/40ede31b/server/pom.xml
--
diff --git a/server/pom.xml b/server/pom.xml
index 981d64c..ed96442 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -229,6 +229,10 @@
 javax.servlet.jsp
 jsp-api
 
+
+jetty-all
+org.eclipse.jetty.aggregate
+
 
 
 

[46/50] [abbrv] kylin git commit: minor, restore @Ignore to ITKylinQueryTest

[lidong]

minor, restore @Ignore to ITKylinQueryTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c9ab289c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c9ab289c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c9ab289c

Branch: refs/heads/master-hbase1.x
Commit: c9ab289c09bf95b97d67315f5d7e0b9d634aa48b
Parents: be11dc6
Author: Yang Li 
Authored: Wed Jan 11 21:11:47 2017 +0800
Committer: Yang Li 
Committed: Wed Jan 11 21:11:47 2017 +0800

--
 .../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c9ab289c/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index aaf0925..82d0873 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -47,7 +47,7 @@ import org.junit.rules.ExpectedException;
 
 import com.google.common.collect.Maps;
 
-//@Ignore("KylinQueryTest is contained by ITCombinationTest")
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
 public class ITKylinQueryTest extends KylinTestBase {
 
 @Rule

[10/50] [abbrv] kylin git commit: KYLIN-2351 Support using non-default HDFS path as kylin.env.hdfs-working-dir

[lidong]

KYLIN-2351 Support using non-default HDFS path as kylin.env.hdfs-working-dir


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/399f0c45
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/399f0c45
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/399f0c45

Branch: refs/heads/master-hbase1.x
Commit: 399f0c45108115c627cbf5f452ec33ff94167247
Parents: 503d232
Author: Li Yang 
Authored: Wed Jan 4 18:24:48 2017 +0800
Committer: Yang Li 
Committed: Fri Jan 6 06:45:37 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  9 +++
 .../common/util/LocalFileMetadataTestCase.java  |  4 ++-
 .../kylin/dict/AppendTrieDictionaryTest.java|  2 +-
 .../org/apache/kylin/engine/mr/HadoopUtil.java  | 28 +++-
 .../engine/mr/common/AbstractHadoopJob.java |  4 +--
 .../engine/mr/steps/CreateDictionaryJob.java|  2 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 10 ---
 .../engine/mr/steps/MergeStatisticsStep.java|  2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  |  3 ++-
 .../engine/mr/steps/SaveStatisticsStep.java |  2 +-
 .../kylin/engine/mr/SortedColumnReaderTest.java | 20 +-
 .../kylin/provision/BuildCubeWithEngine.java|  3 +--
 .../apache/kylin/query/ITMassInQueryTest.java   |  3 +--
 .../kylin/storage/hbase/ITHdfsOpsTest.java  |  4 +--
 .../apache/kylin/source/hive/HiveMRInput.java   |  2 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |  2 +-
 .../kylin/storage/hbase/HBaseResourceStore.java | 11 +++-
 .../storage/hbase/steps/DeprecatedGCStep.java   |  3 +--
 .../steps/HDFSPathGarbageCollectionStep.java|  2 +-
 .../hbase/steps/SequenceFileCuboidWriter.java   |  2 +-
 .../storage/hbase/util/CubeMigrationCLI.java|  3 ++-
 .../storage/hbase/util/StorageCleanupJob.java   |  3 ++-
 .../org/apache/kylin/tool/CubeMigrationCLI.java |  3 ++-
 .../apache/kylin/tool/StorageCleanupJob.java|  4 +--
 24 files changed, 87 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3e6384f..88c4dbc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -171,6 +171,15 @@ abstract public class KylinConfigBase implements 
Serializable {
 if (!root.endsWith("/")) {
 root += "/";
 }
+
+// make sure path qualified
+if (!root.contains("://")) {
+if (!root.startsWith("/"))
+root = "hdfs:///" + root;
+else
+root = "hdfs://" + root;
+}
+
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index 26e1e86..4a24941 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -57,7 +57,9 @@ public class LocalFileMetadataTestCase extends 
AbstractKylinTestCase {
 if (System.getProperty(KylinConfig.KYLIN_CONF) == null && 
System.getenv(KylinConfig.KYLIN_CONF) == null)
 System.setProperty(KylinConfig.KYLIN_CONF, tempTestMetadataUrl);
 
-KylinConfig.getInstanceFromEnv().setMetadataUrl(tempTestMetadataUrl);
+KylinConfig config = KylinConfig.getInstanceFromEnv();
+config.setMetadataUrl(tempTestMetadataUrl);
+config.setProperty("kylin.env.hdfs-working-dir", "file:///tmp/kylin");
 }
 
 public static void cleanAfterClass() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
 
b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index f84f08f..921925c 100644
--- 

[11/50] [abbrv] kylin git commit: KYLIN-2351 fix checkstyle

[lidong]

KYLIN-2351 fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f5734861
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f5734861
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f5734861

Branch: refs/heads/master-hbase1.x
Commit: f5734861863c506510ed23bc1b3891d566b7219d
Parents: 399f0c4
Author: Billy Liu 
Authored: Fri Jan 6 09:15:38 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 6 09:15:38 2017 +0800

--
 .../src/test/java/org/apache/kylin/query/ITMassInQueryTest.java | 1 -
 .../src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java | 1 -
 2 files changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f5734861/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 18d79ae..fd3abfb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -26,7 +26,6 @@ import java.util.Random;
 import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/kylin/blob/f5734861/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
index 786d7d1..605b79e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
@@ -20,7 +20,6 @@ package org.apache.kylin.storage.hbase;
 
 import java.io.IOException;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

[34/50] [abbrv] kylin git commit: KYLIN-2331 by layer spark cubing

[lidong]

KYLIN-2331 by layer spark cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b7d87bb4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b7d87bb4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b7d87bb4

Branch: refs/heads/master-hbase1.x
Commit: b7d87bb4e6c4de484d910e9af04965392da518e5
Parents: 75625ae
Author: shaofengshi 
Authored: Sun Dec 25 15:59:16 2016 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 assembly/pom.xml|   5 +-
 .../main/config/assemblies/source-assembly.xml  |   3 -
 .../java/org/apache/kylin/job/DeployUtil.java   |   1 -
 build/conf/kylin-spark-conf.properties  |  26 ++
 build/conf/kylin.properties |  20 +
 .../apache/kylin/common/KylinConfigBase.java|  64 ++-
 .../kylin/common/persistence/ResourceStore.java |  35 ++
 .../org/apache/kylin/common/util/Array.java |   2 +-
 .../apache/kylin/common/util/SplittedBytes.java |   2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   2 +-
 .../kylin/cube/common/RowKeySplitter.java   |  15 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java|  27 +-
 .../kylin/cube/cuboid/CuboidScheduler.java  |  15 +-
 .../cube/inmemcubing/InMemCubeBuilder.java  |   2 +
 .../kylin/cube/kv/AbstractRowKeyEncoder.java|   6 +-
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |   7 +-
 .../apache/kylin/cube/kv/RowKeyColumnIO.java|   2 +-
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java |  19 +-
 .../kylin/cube/kv/RowKeyEncoderProvider.java|   2 +-
 .../kylin/cube/model/AggregationGroup.java  |   2 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |   2 +-
 .../cube/model/CubeJoinedFlatTableEnrich.java   |   6 +-
 .../apache/kylin/cube/model/DictionaryDesc.java |   4 +-
 .../apache/kylin/cube/model/DimensionDesc.java  |  14 +-
 .../kylin/cube/model/HBaseColumnDesc.java   |  11 +-
 .../kylin/cube/model/HBaseColumnFamilyDesc.java |   9 +-
 .../kylin/cube/model/HBaseMappingDesc.java  |  10 +-
 .../apache/kylin/cube/model/HierarchyDesc.java  |   2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |   2 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java |  18 +-
 .../org/apache/kylin/cube/model/SelectRule.java |   2 +-
 .../apache/kylin/gridtable/UnitTestSupport.java |  41 +-
 .../gridtable/AggregationCacheMemSizeTest.java  |   6 +-
 .../gridtable/AggregationCacheSpillTest.java|   5 +-
 .../kylin/gridtable/SimpleGridTableTest.java|  11 +-
 .../gridtable/SimpleInvertedIndexTest.java  |   3 +-
 .../metadata/measure/MeasureCodecTest.java  |   6 +-
 .../apache/kylin/dict/AppendTrieDictionary.java |   6 +-
 .../org/apache/kylin/dict/NumberDictionary.java |   2 +-
 .../apache/kylin/dict/NumberDictionary2.java|   2 +-
 .../dict/NumberDictionaryForestBuilder.java |   4 +-
 .../apache/kylin/dict/StringBytesConverter.java |   2 +-
 .../kylin/job/execution/ExecutableManager.java  |  21 +-
 .../kylin/dimension/AbstractDateDimEnc.java |  12 +-
 .../apache/kylin/dimension/BooleanDimEnc.java   |   4 +-
 .../kylin/dimension/DictionaryDimEnc.java   |  21 +-
 .../apache/kylin/dimension/FixedLenDimEnc.java  |   4 +-
 .../kylin/dimension/FixedLenHexDimEnc.java  |   4 +-
 .../org/apache/kylin/dimension/IntDimEnc.java   |   4 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   4 +-
 .../kylin/dimension/OneMoreByteVLongDimEnc.java |  14 +-
 .../kylin/measure/BufferedMeasureCodec.java |  10 +-
 .../apache/kylin/measure/MeasureAggregator.java |   2 +
 .../kylin/measure/MeasureAggregators.java   |  18 +
 .../org/apache/kylin/measure/MeasureCodec.java  |   8 +-
 .../apache/kylin/measure/MeasureIngester.java   |  12 +-
 .../org/apache/kylin/measure/MeasureType.java   |  12 +-
 .../measure/basic/BigDecimalMaxAggregator.java  |  14 +
 .../measure/basic/BigDecimalMinAggregator.java  |  28 +-
 .../measure/basic/BigDecimalSumAggregator.java  |  10 +
 .../kylin/measure/basic/DoubleIngester.java |  19 +-
 .../measure/basic/DoubleMaxAggregator.java  |  20 +-
 .../measure/basic/DoubleMinAggregator.java  |  20 +-
 .../measure/basic/DoubleSumAggregator.java  |  18 +-
 .../kylin/measure/basic/LongIngester.java   |  19 +-
 .../kylin/measure/basic/LongMaxAggregator.java  |  20 +-
 .../kylin/measure/basic/LongMinAggregator.java  |  20 +-
 .../kylin/measure/basic/LongSumAggregator.java  |  18 +-
 .../kylin/measure/bitmap/BitmapAggregator.java  |  13 +
 .../kylin/measure/bitmap/BitmapCounter.java |   2 +-
 .../kylin/measure/bitmap/BitmapMeasureType.java |   5 +
 .../ExtendedColumnMeasureType.java  |  15 +
 .../kylin/measure/hllc/DenseRegister.java   |   2 +-
 .../kylin/measure/hllc/HLLCAggregator.java  |   7 +
 

[12/50] [abbrv] kylin git commit: KYLIN-2361 Upgrade to Tomcat 8.5.9

[lidong]

KYLIN-2361 Upgrade to Tomcat 8.5.9


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f267ba9d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f267ba9d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f267ba9d

Branch: refs/heads/master-hbase1.x
Commit: f267ba9dcb3db317d07eb367ddff668e03291519
Parents: f573486
Author: Billy Liu 
Authored: Fri Jan 6 10:42:48 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 6 10:42:48 2017 +0800

--
 build/deploy/context.xml| 47 ++---
 build/deploy/server.xml | 72 +---
 build/script/download-tomcat.sh | 14 ++--
 pom.xml |  2 +-
 .../java/org/apache/kylin/rest/DebugTomcat.java | 10 +--
 .../kylin/ext/CustomizedWebappClassloader.java  |  5 +-
 6 files changed, 82 insertions(+), 68 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/build/deploy/context.xml
--
diff --git a/build/deploy/context.xml b/build/deploy/context.xml
index 5d1bedf..e8b4cf1 100644
--- a/build/deploy/context.xml
+++ b/build/deploy/context.xml
@@ -1,38 +1,31 @@
-
+
 
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
 
 
 
-
+
+
 WEB-INF/web.xml
+${catalina.base}/conf/web.xml
 
 
 
-
-
-
-
 
-
-
\ No newline at end of file
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/build/deploy/server.xml
--
diff --git a/build/deploy/server.xml b/build/deploy/server.xml
index 72a87d0..bea6146 100644
--- a/build/deploy/server.xml
+++ b/build/deploy/server.xml
@@ -1,4 +1,4 @@
-
+
 
 
+
 
 
-
-
-
+
 
-
-
-
+
+
+
 
 
 
 -->
-
+
 
+
+
 
 
-
+
 
 
 
 
+   prefix="localhost_access_log" suffix=".txt"
+   pattern="%h %l %u %t %r %s %b" />
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/build/script/download-tomcat.sh
--
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index 9dd26f0..c97c3ce 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -27,22 +27,22 @@ if [[ `uname -a` =~ "Darwin" ]]; then
 alias md5cmd="md5 -q"
 fi
 
-if [ ! -f "build/apache-tomcat-7.0.69.tar.gz" ]
+if [ ! -f "build/apache-tomcat-8.5.9.tar.gz" ]
 then
 echo "no binary file found"
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.69/bin/apache-tomcat-7.0.69.tar.gz
 || echo "download tomcat failed"
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v8.5.9/bin/apache-tomcat-8.5.9.tar.gz
 || echo "download tomcat failed"
 else
-if [ `md5cmd build/apache-tomcat-7.0.69.tar.gz | awk '{print $1}'` != 
"10a071e5169a1a8b14ff35a0ad181052" ]
+if [ `md5cmd build/apache-tomcat-8.5.9.tar.gz | awk '{print $1}'` != 
"b41270a64b7774c964e4bec813eea2ed" ]
 then
 echo "md5 check failed"
-rm build/apache-tomcat-7.0.69.tar.gz
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.69/bin/apache-tomcat-7.0.69.tar.gz
 || echo "download tomcat failed"
+rm build/apache-tomcat-8.5.9.tar.gz
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v8.5.9/bin/apache-tomcat-8.5.9.tar.g
 || echo "download tomcat failed"
 fi
 fi
 unalias md5cmd
 
-tar -zxvf build/apache-tomcat-7.0.69.tar.gz -C build/
-mv build/apache-tomcat-7.0.69 build/tomcat
+tar -zxvf build/apache-tomcat-8.5.9.tar.gz -C build/
+mv build/apache-tomcat-8.5.9 build/tomcat
 rm -rf build/tomcat/webapps/*
 
 mv 

[32/50] [abbrv] kylin git commit: KYLIN-2331 by layer spark cubing

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
new file mode 100644
index 000..07b636b
--- /dev/null
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import com.google.common.collect.Sets;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ */
+public class BaseCuboidBuilder implements java.io.Serializable {
+
+protected static final Logger logger = 
LoggerFactory.getLogger(BaseCuboidBuilder.class);
+public static final String HIVE_NULL = "\\N";
+protected String cubeName;
+protected Cuboid baseCuboid;
+protected CubeDesc cubeDesc;
+protected CubeSegment cubeSegment;
+protected Set nullStrs;
+protected CubeJoinedFlatTableEnrich intermediateTableDesc;
+protected MeasureIngester[] aggrIngesters;
+protected Map dictionaryMap;
+protected AbstractRowKeyEncoder rowKeyEncoder;
+protected BufferedMeasureCodec measureCodec;
+
+protected KylinConfig kylinConfig;
+
+public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, 
CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc,
+ AbstractRowKeyEncoder rowKeyEncoder, 
MeasureIngester[] aggrIngesters, Map 
dictionaryMap) {
+this.kylinConfig = kylinConfig;
+this.cubeDesc = cubeDesc;
+this.cubeSegment = cubeSegment;
+this.intermediateTableDesc = intermediateTableDesc;
+this.rowKeyEncoder = rowKeyEncoder;
+this.aggrIngesters = aggrIngesters;
+this.dictionaryMap = dictionaryMap;
+
+init();
+measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+}
+
+public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, 
CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc) {
+this.kylinConfig = kylinConfig;
+this.cubeDesc = cubeDesc;
+this.cubeSegment = cubeSegment;
+this.intermediateTableDesc = intermediateTableDesc;
+
+init();
+rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, 
baseCuboid);
+measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+aggrIngesters = MeasureIngester.create(cubeDesc.getMeasures());
+dictionaryMap = cubeSegment.buildDictionaryMap();
+
+}
+
+private void init() {
+long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+initNullBytes();
+}
+
+private void initNullBytes() {
+nullStrs = Sets.newHashSet();
+nullStrs.add(HIVE_NULL);
+String[] nullStrings = cubeDesc.getNullStrings();
+if (nullStrings != null) {
+for (String s : nullStrings) {
+nullStrs.add(s);
+}
+}
+}
+
+protected boolean isNull(String v) {
+return nullStrs.contains(v);
+}
+
+public byte[] buildKey(String[] 

[41/50] [abbrv] kylin git commit: KYLIN-2379 Add UseCMSInitiatingOccupancyOnly to KYLIN_JVM_SETTINGS

[lidong]

KYLIN-2379 Add UseCMSInitiatingOccupancyOnly to KYLIN_JVM_SETTINGS

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c84b46bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c84b46bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c84b46bd

Branch: refs/heads/master-hbase1.x
Commit: c84b46bd5b72006f140f7aad147ba0185c7eecbb
Parents: 07e5e5a
Author: kangkaisen 
Authored: Tue Jan 10 20:32:39 2017 +0800
Committer: Billy Liu 
Committed: Tue Jan 10 20:48:43 2017 +0800

--
 build/bin/setenv.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c84b46bd/build/bin/setenv.sh
--
diff --git a/build/bin/setenv.sh b/build/bin/setenv.sh
index f6cf210..d838362 100755
--- a/build/bin/setenv.sh
+++ b/build/bin/setenv.sh
@@ -22,7 +22,7 @@
 # (if your're deploying KYLIN on a powerful server and want to replace the 
default conservative settings)
 # uncomment following to for it to take effect
 export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -Xss1024K -XX:MaxPermSize=128M 
-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps 
-Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation 
-XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M"
-# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m 
-XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 
-XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled 
-XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode 
-XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC 
-XX:+HeapDumpOnOutOfMemoryError"
+# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m 
-XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 
-XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled 
-XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode 
-XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly 
-XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError"
 
 # uncomment following to for it to take effect(the values need adjusting to 
fit your env)
 # export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote 
-Dcom.sun.management.jmxremote.authenticate=false 
-Dcom.sun.management.jmxremote.ssl=false"

[35/50] [abbrv] kylin git commit: KYLIN-2344 Package spark into Kylin binary package

[lidong]

KYLIN-2344 Package spark into Kylin binary package


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e864cd3b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e864cd3b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e864cd3b

Branch: refs/heads/master-hbase1.x
Commit: e864cd3b1c06700b1a1054d1f520eaabedc25d82
Parents: b7d87bb
Author: shaofengshi 
Authored: Tue Jan 3 10:28:38 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 build/bin/check-env.sh  |  6 ++
 build/bin/find-spark-dependency.sh  | 45 +++
 build/bin/kylin.sh  |  6 ++
 build/conf/kylin-spark-conf.properties  |  2 +-
 build/conf/kylin.properties |  3 -
 build/script/compress.sh|  5 +-
 build/script/download-spark.sh  | 52 +
 build/script/functions.sh   | 60 
 build/script/package.sh |  1 +
 .../org/apache/kylin/common/KylinConfig.java| 14 +
 .../apache/kylin/common/KylinConfigBase.java| 27 -
 .../spark/SparkBatchCubingJobBuilder2.java  |  4 +-
 12 files changed, 200 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/bin/check-env.sh
--
diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh
index a4003c9..e446d66 100644
--- a/build/bin/check-env.sh
+++ b/build/bin/check-env.sh
@@ -47,3 +47,9 @@ if [ $? != 0 ]
 then
 quit "Failed to create $WORKING_DIR. Please make sure the user has right 
to access $WORKING_DIR"
 fi
+
+hadoop fs -mkdir -p $WORKING_DIR/spark-history
+if [ $? != 0 ]
+then
+quit "Failed to create $WORKING_DIR/spark-history. Please make sure the 
user has right to access $WORKING_DIR"
+fi
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/bin/find-spark-dependency.sh
--
diff --git a/build/bin/find-spark-dependency.sh 
b/build/bin/find-spark-dependency.sh
new file mode 100644
index 000..6f74d8a
--- /dev/null
+++ b/build/bin/find-spark-dependency.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
+spark_home=
+
+verbose Retrieving Spark dependency...
+if [ -n "$SPARK_HOME" ]
+then
+verbose "SPARK_HOME is set to: $SPARK_HOME, use it to locate Spark 
dependencies."
+spark_home=$SPARK_HOME
+fi
+
+if [ -z "$SPARK_HOME" ]
+then
+verbose "SPARK_HOME wasn't set, use $KYLIN_HOME/spark"
+spark_home=$KYLIN_HOME/spark
+fi
+
+spark_dependency=`find -L $spark_home -name 
'spark-assembly-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name 
'*sources*' ''-printf '%p:' | sed 's/:$//'`
+if [ -z "$spark_dependency" ]
+then
+quit "spark assembly lib not found"
+else
+verbose "spark dependency: $spark_dependency"
+export spark_dependency
+fi
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 0cdbbc6..7813b79 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -45,6 +45,11 @@ function retrieveDependency() {
 source ${dir}/find-kafka-dependency.sh
 export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
 fi
+
+source ${dir}/find-spark-dependency.sh
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${spark_dependency}
+
+verbose "HBASE_CLASSPATH: ${HBASE_CLASSPATH}"
 }
 
 # start command
@@ -112,6 +117,7 @@ then
 -Dkylin.hive.dependency=${hive_dependency} \
 -Dkylin.hbase.dependency=${hbase_dependency} \
 -Dkylin.kafka.dependency=${kafka_dependency} \
+-Dkylin.spark.dependency=${spark_dependency} \

[24/50] [abbrv] kylin git commit: KYLIN-2353 minor code review

[lidong]

KYLIN-2353 minor code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56daf6d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56daf6d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56daf6d5

Branch: refs/heads/master-hbase1.x
Commit: 56daf6d592ae454b3bca97c4a7a948781485befa
Parents: 731a53a
Author: Yang Li 
Authored: Sat Jan 7 19:30:27 2017 +0800
Committer: Yang Li 
Committed: Sat Jan 7 19:30:27 2017 +0800

--
 .../org/apache/kylin/measure/bitmap/BitmapCounter.java| 10 ++
 1 file changed, 6 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56daf6d5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index a18ac4a..a632b0d 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -51,7 +51,9 @@ public class BitmapCounter implements 
Comparable {
 }
 
 if (buffer != null) {
+@SuppressWarnings("unused")
 int version = buffer.getInt();
+@SuppressWarnings("unused")
 int size = buffer.getInt();
 count = buffer.getInt();
 
@@ -79,7 +81,7 @@ public class BitmapCounter implements 
Comparable {
 
 public void add(int value) {
 getBitmap().add(value);
-count = getBitmap().getCardinality();
+count = null;
 }
 
 public void add(byte[] value) {
@@ -103,12 +105,12 @@ public class BitmapCounter implements 
Comparable {
 
 public void merge(BitmapCounter another) {
 getBitmap().or(another.getBitmap());
-count = getBitmap().getCardinality();
+count = null;
 }
 
 public void intersect(BitmapCounter another) {
 getBitmap().and(another.getBitmap());
-count = getBitmap().getCardinality();
+count = null;
 }
 
 public int getCount() {
@@ -148,7 +150,6 @@ public class BitmapCounter implements 
Comparable {
 
 // keep forward compatibility
 if (version == VERSION) {
-@SuppressWarnings("unused")
 int size = in.getInt();
 count = in.getInt();
 in.position(mark);
@@ -258,4 +259,5 @@ public class BitmapCounter implements 
Comparable {
 return len;
 }
 }
+
 }

[29/50] [abbrv] kylin git commit: fix NPE in CacheDictionary in Spark cubing

[lidong]

fix NPE in CacheDictionary in Spark cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/66bca9a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/66bca9a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/66bca9a6

Branch: refs/heads/master-hbase1.x
Commit: 66bca9a676fda02aa88cc25bad2e545f779f6dde
Parents: 2cf52b4
Author: shaofengshi 
Authored: Sat Jan 7 10:17:33 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 .../java/org/apache/kylin/dict/CacheDictionary.java   |  2 +-
 .../apache/kylin/engine/mr/BatchCubingJobBuilder.java |  5 ++---
 .../kylin/engine/mr/BatchCubingJobBuilder2.java   |  7 +++
 .../org/apache/kylin/engine/mr/JobBuilderSupport.java | 14 +-
 .../engine/spark/SparkBatchCubingJobBuilder2.java |  1 +
 .../apache/kylin/engine/spark/SparkCubingByLayer.java |  2 +-
 .../kylin/engine/spark/SparkCubingJobBuilder.java |  1 -
 7 files changed, 9 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
index d7ed6bd..b2bad53 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
@@ -35,7 +35,7 @@ public abstract class CacheDictionary extends 
Dictionary {
 
 protected transient int baseId;
 
-protected transient BytesConverter bytesConvert;
+protected BytesConverter bytesConvert;
 
 public CacheDictionary() {
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 456f615..36c12a1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -65,12 +65,11 @@ public class BatchCubingJobBuilder extends 
JobBuilderSupport {
 // Phase 3: Build Cube
 RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey();
 final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel();
-final String[] cuboidOutputTempPath = 
getCuboidOutputPaths(cuboidRootPath, groupRowkeyColumnsCount);
 // base cuboid step
-result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
+
result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 
0), jobId));
 // n dim cuboid steps
 for (int i = 1; i <= groupRowkeyColumnsCount; i++) {
-result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 
1], cuboidOutputTempPath[i], i));
+
result.addTask(createNDimensionCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath,
 i - 1), getCuboidOutputPathsByLevel(cuboidRootPath, i), i));
 }
 outputSide.addStepPhase3_BuildCube(result, cuboidRootPath);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 700f821..dd866bd 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -76,14 +76,13 @@ public class BatchCubingJobBuilder2 extends 
JobBuilderSupport {
 return result;
 }
 
-private void addLayerCubingSteps(final CubingJob result, final String 
jobId, final String cuboidRootPath) {
+protected void addLayerCubingSteps(final CubingJob result, final String 
jobId, final String cuboidRootPath) {
 final int maxLevel = seg.getCubeDesc().getBuildLevel();
-final String[] cuboidOutputTempPath = 
getCuboidOutputPaths(cuboidRootPath, maxLevel);
 // base cuboid step
-result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
+
result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 
0), jobId));
 // n dim cuboid steps
 

[02/50] [abbrv] kylin git commit: KYLIN-2359 Update job build step name

[lidong]

KYLIN-2359 Update job build step name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c985742
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c985742
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c985742

Branch: refs/heads/master-hbase1.x
Commit: 9c985742584a6bf15053d5482d13e1e01f76894d
Parents: 809b783
Author: shaofengshi 
Authored: Thu Jan 5 12:19:42 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 5 12:19:42 2017 +0800

--
 .../org/apache/kylin/job/constant/ExecutableConstants.java| 7 ---
 .../org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java| 4 ++--
 .../org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java| 4 ++--
 3 files changed, 8 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9c985742/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 4ae3add..11c7455 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -38,9 +38,9 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create 
Intermediate Flat Hive Table";
 public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = 
"Materialize Hive View in Lookup Tables";
 public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact 
Table Distinct Columns";
-public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base 
Cuboid Data";
-public static final String STEP_NAME_BUILD_IN_MEM_CUBE = "Build Cube";
-public static final String STEP_NAME_BUILD_N_D_CUBOID = "Build N-Dimension 
Cuboid Data";
+public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base 
Cuboid";
+public static final String STEP_NAME_BUILD_IN_MEM_CUBE = "Build Cube 
In-Mem";
+public static final String STEP_NAME_BUILD_N_D_CUBOID = "Build N-Dimension 
Cuboid";
 public static final String STEP_NAME_GET_CUBOID_KEY_DISTRIBUTION = 
"Calculate HTable Region Splits";
 public static final String STEP_NAME_CREATE_HBASE_TABLE = "Create HTable";
 public static final String STEP_NAME_CONVERT_CUBOID_TO_HFILE = "Convert 
Cuboid Data to HFile";
@@ -53,6 +53,7 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_HIVE_CLEANUP = "Hive Cleanup";
 public static final String STEP_NAME_KAFKA_CLEANUP = "Kafka Intermediate 
File Cleanup";
 public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage 
Collection";
+public static final String STEP_NAME_GARBAGE_COLLECTION_HBASE = "Garbage 
Collection on HBase";
 public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage 
Collection on HDFS";
 public static final String STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE = 
"Redistribute Flat Hive Table";
 public static final String NOTIFY_EMAIL_TEMPLATE = "Build Result 
of Job ${job_name}" + "Build Result: ${result}" + 
"Job Engine: ${job_engine}" + "Env: ${env_name}" + 
"Project: ${project_name}" + "Cube Name: ${cube_name}" + 
"Source Records Count: ${source_records_count}" + "Start Time: 
${start_time}" + "Duration: ${duration}" + "MR Waiting: 
${mr_waiting}" + "Last Update Time: ${last_update_time}" + 
"Submitter: ${submitter}" + "Error Log: ${error_log}" + 
"";

http://git-wip-us.apache.org/repos/asf/kylin/blob/9c985742/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 20540a6..9229a75 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -43,13 +43,13 @@ public class BatchCubingJobBuilder2 extends 
JobBuilderSupport {
 public BatchCubingJobBuilder2(CubeSegment newSegment, String submitter) {
 super(newSegment, submitter);
 this.inputSide = MRUtil.getBatchCubingInputSide(seg);
-this.outputSide = MRUtil.getBatchCubingOutputSide2((CubeSegment) seg);
+this.outputSide = MRUtil.getBatchCubingOutputSide2(seg);
 }
 
 public CubingJob build() {
 logger.info("MR_V2 new job to BUILD segment " + seg);
 
-final CubingJob result = 

[42/50] [abbrv] kylin git commit: KYLIN-2376 upgrade checkstyle plugin

[lidong]

KYLIN-2376 upgrade checkstyle plugin

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/521dfffa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/521dfffa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/521dfffa

Branch: refs/heads/master-hbase1.x
Commit: 521dfffae15d0c85885ec816a4fc40907bfb7645
Parents: c84b46b
Author: etherge 
Authored: Tue Jan 10 19:09:58 2017 +0800
Committer: Billy Liu 
Committed: Tue Jan 10 20:53:57 2017 +0800

--
 pom.xml | 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/521dfffa/pom.xml
--
diff --git a/pom.xml b/pom.xml
index fcb5ce3..e391103 100644
--- a/pom.xml
+++ b/pom.xml
@@ -907,6 +907,13 @@
 org.apache.maven.plugins
 maven-checkstyle-plugin
 2.17
+
+
+com.puppycrawl.tools
+checkstyle
+6.19
+
+
 
 
 check-style

[20/50] [abbrv] kylin git commit: fix NPE bug

[lidong]

fix NPE bug

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5a18af04
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5a18af04
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5a18af04

Branch: refs/heads/master-hbase1.x
Commit: 5a18af04ca40aaa6509cdf7c39b71b17dc5eb4a3
Parents: 122b15a
Author: xiefan46 <958034...@qq.com>
Authored: Thu Jan 5 23:29:43 2017 +0800
Committer: shaofengshi 
Committed: Sat Jan 7 10:18:20 2017 +0800

--
 .../src/main/java/org/apache/kylin/dict/CacheDictionary.java | 8 ++--
 1 file changed, 2 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5a18af04/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
index 1e260b2..d7ed6bd 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
@@ -29,8 +29,6 @@ import java.util.concurrent.ConcurrentHashMap;
 public abstract class CacheDictionary extends Dictionary {
 private static final long serialVersionUID = 1L;
 
-protected transient boolean enableValueCache = false;
-
 private transient SoftReference valueToIdCache;
 
 private transient SoftReference idToValueCache;
@@ -46,7 +44,7 @@ public abstract class CacheDictionary extends 
Dictionary {
 //value --> id
 @Override
 protected final int getIdFromValueImpl(T value, int roundingFlag) {
-if (enableValueCache && roundingFlag == 0) {
+if (this.valueToIdCache != null && roundingFlag == 0) {
 Map cache = valueToIdCache.get(); // SoftReference to skip cache 
gracefully when short of memory
 if (cache != null) {
 Integer id;
@@ -66,7 +64,7 @@ public abstract class CacheDictionary extends 
Dictionary {
 //id --> value
 @Override
 protected final T getValueFromIdImpl(int id) {
-if (enableValueCache) {
+if (this.idToValueCache != null) {
 Object[] cache = idToValueCache.get();
 if (cache != null) {
 int seq = calcSeqNoFromId(id);
@@ -91,7 +89,6 @@ public abstract class CacheDictionary extends 
Dictionary {
 }
 
 public final void enableCache() {
-this.enableValueCache = true;
 if (this.valueToIdCache == null)
 this.valueToIdCache = new SoftReference<>(new ConcurrentHashMap());
 if (this.idToValueCache == null)
@@ -99,7 +96,6 @@ public abstract class CacheDictionary extends 
Dictionary {
 }
 
 public final void disableCache() {
-this.enableValueCache = false;
 this.valueToIdCache = null;
 this.idToValueCache = null;
 }

[40/50] [abbrv] kylin git commit: KYLIN-2367 ammend test case

[lidong]

KYLIN-2367 ammend test case


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07e5e5a7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07e5e5a7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07e5e5a7

Branch: refs/heads/master-hbase1.x
Commit: 07e5e5a75f4f692a53ac48c64c3cb44b84022c88
Parents: b0aadb1
Author: Li Yang 
Authored: Tue Jan 10 15:45:05 2017 +0800
Committer: Li Yang 
Committed: Tue Jan 10 15:45:05 2017 +0800

--
 .../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/07e5e5a7/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 2b33e66..df53de9 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -48,7 +48,7 @@
   }, {
 "id" : "10",
 "name" : "ITEM_COUNT",
-"datatype" : "int",
+"datatype" : "bigint",
 "data_gen" : "RAND"
   }, {
 "id" : "11",

[50/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c47f4ad5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c47f4ad5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c47f4ad5

Branch: refs/heads/master-hbase1.x
Commit: c47f4ad5767e7c9604b78bbae798805ce22e02fe
Parents: b0a406f
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Thu Jan 12 12:02:26 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 62978db..e46cbe1 100644
--- 

[26/50] [abbrv] kylin git commit: minor, enable jacoco plugin

[lidong]

minor, enable jacoco plugin


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68ff75de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68ff75de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68ff75de

Branch: refs/heads/master-hbase1.x
Commit: 68ff75de260e6e93197103c0fd06009cf73c8e21
Parents: e46d699
Author: Billy Liu 
Authored: Mon Jan 9 00:29:57 2017 +0800
Committer: Billy Liu 
Committed: Mon Jan 9 00:29:57 2017 +0800

--
 pom.xml | 47 +++
 1 file changed, 31 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/68ff75de/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 22681e8..d883eca 100644
--- a/pom.xml
+++ b/pom.xml
@@ -922,19 +922,7 @@
 
 org.jacoco
 jacoco-maven-plugin
-0.7.7.201606060606
-
-${sonar.jacoco.reportPath}
-true
-
-
-
-agent
-
-prepare-agent
-
-
-
+0.7.8
 
 
 org.apache.maven.plugins
@@ -1079,6 +1067,35 @@
 
 
 
+org.jacoco
+jacoco-maven-plugin
+
+true
+
+${sonar.jacoco.reportPath}
+
+
+
+
+pre-test
+
+prepare-agent
+
+
+
surefireArgLine
+
+
+
+post-test
+test
+
+report
+
+
+
+
+
+
 org.apache.maven.plugins
 maven-surefire-plugin
 2.19.1
@@ -1101,11 +1118,9 @@
 
file:${project.basedir}/../build/conf/kylin-tools-log4j.properties
 
 
-
-javaagent:${project.build.testOutputDirectory}/jamm.jar 
${argLine}
+
-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} 
${surefireArgLine}
 
 
-
-
 
 
 

[43/50] [abbrv] kylin git commit: rollback ITEM_COUNT to INT type

[lidong]

rollback ITEM_COUNT to INT type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78d7086e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78d7086e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78d7086e

Branch: refs/heads/master-hbase1.x
Commit: 78d7086e51c9311ab25b17f05206766d8a279afa
Parents: 521dfff
Author: Li Yang 
Authored: Wed Jan 11 09:40:55 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 09:40:55 2017 +0800

--
 .../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/78d7086e/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index df53de9..2b33e66 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -48,7 +48,7 @@
   }, {
 "id" : "10",
 "name" : "ITEM_COUNT",
-"datatype" : "bigint",
+"datatype" : "int",
 "data_gen" : "RAND"
   }, {
 "id" : "11",

[08/50] [abbrv] kylin git commit: KYLIN-2356 Incorrect result when filter on numeric columns

[lidong]

KYLIN-2356 Incorrect result when filter on numeric columns

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/980b03b1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/980b03b1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/980b03b1

Branch: refs/heads/master-hbase1.x
Commit: 980b03b1abb8af82a9bceeb497847fe17c19c3e3
Parents: c64f55b
Author: zhengdong 
Authored: Thu Jan 5 15:05:35 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 5 17:45:20 2017 +0800

--
 .../filter/FilterCodeSystemFactory.java | 98 
 .../translate/DerivedFilterTranslator.java  |  4 +-
 2 files changed, 100 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/980b03b1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
new file mode 100755
index 000..bae8cf9
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+
+/**
+ * Created by donald.zheng on 2016/12/19.
+ */
+public class FilterCodeSystemFactory {
+private final static HashMap codeSystemMap = 
new HashMap<>();
+
+static {
+codeSystemMap.put("string", StringCodeSystem.INSTANCE);
+codeSystemMap.put("integer", new IntegerCodeSystem());
+codeSystemMap.put("decimal", new DecimalCodeSystem());
+}
+
+public static IFilterCodeSystem getFilterCodeSystem(DataType dataType) {
+if (dataType.isIntegerFamily()) {
+return codeSystemMap.get("integer");
+} else if (dataType.isNumberFamily()) {
+return codeSystemMap.get("decimal");
+} else {
+return codeSystemMap.get("string");
+}
+}
+
+private static class IntegerCodeSystem implements IFilterCodeSystem {
+
+@Override
+public boolean isNull(Object code) {
+return code == null;
+}
+
+@Override
+public void serialize(Object code, ByteBuffer buf) {
+BytesUtil.writeLong(Long.parseLong(code.toString()), buf);
+}
+
+@Override
+public Object deserialize(ByteBuffer buf) {
+return BytesUtil.readLong(buf);
+}
+
+@Override
+public int compare(Object o, Object t1) {
+long l1 = Long.parseLong(o.toString());
+long l2 = Long.parseLong(t1.toString());
+return Long.compare(l1, l2);
+}
+}
+
+private static class DecimalCodeSystem implements IFilterCodeSystem {
+@Override
+public boolean isNull(Object code) {
+return code == null;
+}
+
+@Override
+public void serialize(Object code, ByteBuffer buf) {
+BytesUtil.writeUTFString(code.toString(), buf);
+}
+
+@Override
+public Object deserialize(ByteBuffer buf) {
+return Double.parseDouble(BytesUtil.readUTFString(buf));
+}
+
+@Override
+public int compare(Object o, Object t1) {
+double d1 = Double.parseDouble(o.toString());
+double d2 = Double.parseDouble(t1.toString());
+return Double.compare(d1, d2);
+}
+}
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/980b03b1/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java

[17/50] [abbrv] kylin git commit: minor, update jetty to 9.2.20

[lidong]

minor, update jetty to 9.2.20


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1087f7ad
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1087f7ad
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1087f7ad

Branch: refs/heads/master-hbase1.x
Commit: 1087f7ad49f324b6ee9c1d48b4b6b0489236fe9c
Parents: 1b85f71
Author: Billy Liu 
Authored: Fri Jan 6 16:42:46 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 6 16:43:04 2017 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1087f7ad/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 1d47361..22681e8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,7 +71,7 @@
 1.1.0
 2.5.2
 1.4.192
-9.3.10.v20160621
+9.2.20.v20161216
 0.3.1
 
 

[45/50] [abbrv] kylin git commit: minor, change Dictionary to Dictionary in DictionaryManager and DictionaryInfo

[lidong]

minor,change Dictionary to Dictionary in DictionaryManager and 
DictionaryInfo

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be11dc62
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be11dc62
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be11dc62

Branch: refs/heads/master-hbase1.x
Commit: be11dc629460ef4620300e1508c548037540f961
Parents: 5f6450d
Author: xiefan46 <958034...@qq.com>
Authored: Tue Jan 10 16:33:25 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 13:04:36 2017 +0800

--
 .../java/org/apache/kylin/dict/DictionaryInfo.java|  6 +++---
 .../apache/kylin/dict/DictionaryInfoSerializer.java   |  4 ++--
 .../java/org/apache/kylin/dict/DictionaryManager.java | 14 +++---
 .../kylin/engine/mr/steps/MergeCuboidMapperTest.java  |  6 +++---
 4 files changed, 15 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index 8526467..a85628d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -47,7 +47,7 @@ public class DictionaryInfo extends RootPersistentEntity {
 @JsonProperty("cardinality")
 private int cardinality;
 
-transient Dictionary dictionaryObject;
+transient Dictionary dictionaryObject;
 
 public DictionaryInfo() {
 }
@@ -144,11 +144,11 @@ public class DictionaryInfo extends RootPersistentEntity {
 this.dictionaryClass = dictionaryClass;
 }
 
-public Dictionary getDictionaryObject() {
+public Dictionary getDictionaryObject() {
 return dictionaryObject;
 }
 
-public void setDictionaryObject(Dictionary dictionaryObject) {
+public void setDictionaryObject(Dictionary dictionaryObject) {
 this.dictionaryObject = dictionaryObject;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
index 04eedbe..ee88ea7 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
@@ -61,9 +61,9 @@ public class DictionaryInfoSerializer implements 
Serializer {
 DictionaryInfo obj = JsonUtil.readValue(json, DictionaryInfo.class);
 
 if (infoOnly == false) {
-Dictionary dict;
+Dictionary dict;
 try {
-dict = (Dictionary) 
ClassUtil.forName(obj.getDictionaryClass(), Dictionary.class).newInstance();
+dict = (Dictionary) 
ClassUtil.forName(obj.getDictionaryClass(), Dictionary.class).newInstance();
 } catch (InstantiationException e) {
 throw new RuntimeException(e);
 } catch (IllegalAccessException e) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 54bc1c4..3ba24cf 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -108,7 +108,7 @@ public class DictionaryManager {
 });
 }
 
-public Dictionary getDictionary(String resourcePath) throws IOException 
{
+public Dictionary getDictionary(String resourcePath) throws 
IOException {
 DictionaryInfo dictInfo = getDictionaryInfo(resourcePath);
 return dictInfo == null ? null : dictInfo.getDictionaryObject();
 }
@@ -130,7 +130,7 @@ public class DictionaryManager {
  * Save the dictionary as it is.
  * More often you should consider using its alternative trySaveNewDict to 
save dict space
  */
-public DictionaryInfo forceSave(Dictionary newDict, DictionaryInfo 
newDictInfo) throws IOException {
+public DictionaryInfo forceSave(Dictionary newDict, DictionaryInfo 

[44/50] [abbrv] kylin git commit: KYLIN-2380 refactor DbUnit assertion

[lidong]

KYLIN-2380 refactor DbUnit assertion


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f6450d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f6450d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f6450d6

Branch: refs/heads/master-hbase1.x
Commit: 5f6450d6e77f9f62c3670c20ec4d8aef18bfd2fb
Parents: 78d7086
Author: Li Yang 
Authored: Wed Jan 11 12:10:36 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 12:39:24 2017 +0800

--
 .../apache/kylin/query/HackedDbUnitAssert.java  | 134 ---
 .../apache/kylin/query/ITMassInQueryTest.java   |   9 +-
 .../org/apache/kylin/query/KylinTestBase.java   |  65 +
 3 files changed, 152 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f6450d6/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index c295430..3a21570 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -26,16 +26,34 @@ import org.dbunit.dataset.Columns;
 import org.dbunit.dataset.DataSetException;
 import org.dbunit.dataset.ITable;
 import org.dbunit.dataset.ITableMetaData;
+import org.dbunit.dataset.datatype.BigIntegerDataType;
 import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.IntegerDataType;
+import org.dbunit.dataset.datatype.UnknownDataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * dirty hack to support checking result of SQL with limit
+ * A copy of DbUnitAssert, in which a few hacks applied
+ * 
+ * - tolerate some column type difference, like INT vs. BIGINT
+ * - check expected table contains actual table (instead of equals), for sql 
with limit
  */
 public class HackedDbUnitAssert extends DbUnitAssert {
 private static final Logger logger = 
LoggerFactory.getLogger(HackedDbUnitAssert.class);
 
+private boolean hackCheckContains;
+private boolean hackIgnoreIntBigIntMismatch;
+
+public void hackCheckContains() {
+hackCheckContains = true;
+}
+
+public void hackIgnoreIntBigIntMismatch() {
+hackIgnoreIntBigIntMismatch = true;
+}
+
+// THIS METHOD IS MOSTLY COPIED FROM DbUnitAssert. CHANGES ARE LEAD BY 
hackXXX CONDITION CHECKS.
 public void assertEquals(ITable expectedTable, ITable actualTable, 
FailureHandler failureHandler) throws DatabaseUnitException {
 logger.trace("assertEquals(expectedTable, actualTable, failureHandler) 
- start");
 logger.debug("assertEquals: expectedTable={}", expectedTable);
@@ -57,25 +75,21 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 ITableMetaData actualMetaData = actualTable.getTableMetaData();
 String expectedTableName = expectedMetaData.getTableName();
 
-//// Verify row count
-//int expectedRowsCount = expectedTable.getRowCount();
-//int actualRowsCount = actualTable.getRowCount();
-//if (expectedRowsCount != actualRowsCount) {
-//String msg = "row count (table=" + expectedTableName + 
")";
-//Error error =
-//failureHandler.createFailure(msg, String
-//.valueOf(expectedRowsCount), String
-//.valueOf(actualRowsCount));
-//logger.error(error.toString());
-//throw error;
-//}
-
-// if both tables are empty, it is not necessary to compare columns, as
-// such
-// comparison
-// can fail if column metadata is different (which could occurs when
-// comparing empty tables)
-if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
+// Verify row count
+int expectedRowsCount = expectedTable.getRowCount();
+int actualRowsCount = actualTable.getRowCount();
+if (!hackCheckContains) {
+if (expectedRowsCount != actualRowsCount) {
+String msg = "row count (table=" + expectedTableName + ")";
+Error error = failureHandler.createFailure(msg, 
String.valueOf(expectedRowsCount), String.valueOf(actualRowsCount));
+logger.error(error.toString());
+throw error;
+}
+}
+
+// if both tables are empty, it is not necessary to compare columns, 
as such comparison
+// can fail if column metadata is different 

[01/50] [abbrv] kylin git commit: KYLIN-2342 When NoClassDefFoundError occurred in building cube, no error in kylin.log [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master-hbase1.x 982e8cf2a -> c47f4ad57 (forced update)


KYLIN-2342 When NoClassDefFoundError occurred in building cube, no error in 
kylin.log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e043d5fa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e043d5fa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e043d5fa

Branch: refs/heads/master-hbase1.x
Commit: e043d5fad8ccf420b1c0e744ee87a237cd7babd7
Parents: e6a84c5
Author: shaofengshi 
Authored: Thu Jan 5 10:54:33 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 5 12:01:36 2017 +0800

--
 .../org/apache/kylin/rest/controller/CubeController.java  |  4 ++--
 .../org/apache/kylin/rest/controller/TableController.java | 10 ++
 .../org/apache/kylin/rest/init/InitialTaskManager.java|  2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e043d5fa/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 978f477..a9c8321 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -293,7 +293,7 @@ public class CubeController extends BasicController {
 }
 return jobService.submitJob(cube, startTime, endTime, startOffset, 
endOffset, //
 sourcePartitionOffsetStart, sourcePartitionOffsetEnd, 
CubeBuildTypeEnum.valueOf(buildType), force, submitter);
-} catch (Exception e) {
+} catch (Throwable e) {
 logger.error(e.getLocalizedMessage(), e);
 throw new InternalErrorException(e.getLocalizedMessage());
 }
@@ -656,7 +656,7 @@ public class CubeController extends BasicController {
 cubeService.getCubeDescManager().updateCubeDesc(desc);
 response.setProperty("result", "success");
 response.setProperty("offsets", startOffsets.toString());
-} catch (Exception e) {
+} catch (Throwable e) {
 throw new RuntimeException(e);
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e043d5fa/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 1ab6ba6..ac4ee2f 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -112,7 +112,7 @@ public class TableController extends BasicController {
 if (request.isCalculate()) {
 tableService.calculateCardinalityIfNotPresent(loaded, 
submitter);
 }
-} catch (Exception e) {
+} catch (Throwable e) {
 logger.error("Failed to load Hive Table", e);
 throw new InternalErrorException(e.getLocalizedMessage());
 }
@@ -133,7 +133,7 @@ public class TableController extends BasicController {
 unLoadFail.add(tableName);
 }
 }
-} catch (Exception e) {
+} catch (Throwable e) {
 logger.error("Failed to unload Hive Table", e);
 throw new InternalErrorException(e.getLocalizedMessage());
 }
@@ -175,7 +175,8 @@ public class TableController extends BasicController {
 private List showHiveDatabases() throws IOException {
 try {
 return tableService.getHiveDbNames();
-} catch (Exception e) {
+} catch (Throwable e) {
+logger.error(e.getLocalizedMessage(), e);
 throw new InternalErrorException(e.getLocalizedMessage());
 }
 }
@@ -191,7 +192,8 @@ public class TableController extends BasicController {
 private List showHiveTables(@PathVariable String database) throws 
IOException {
 try {
 return tableService.getHiveTableNames(database);
-} catch (Exception e) {
+} catch (Throwable e) {
+logger.error(e.getLocalizedMessage(), e);
 throw new InternalErrorException(e.getLocalizedMessage());
 }
 }

[25/50] [abbrv] kylin git commit: KYLIN-2364 Output table name to error info in LookupTable

[lidong]

KYLIN-2364 Output table name to error info in LookupTable

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e46d699e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e46d699e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e46d699e

Branch: refs/heads/master-hbase1.x
Commit: e46d699e05100db084db354d7efb3786575d5c54
Parents: 56daf6d
Author: kangkaisen 
Authored: Sat Jan 7 19:43:43 2017 +0800
Committer: Billy Liu 
Committed: Sun Jan 8 12:14:05 2017 +0800

--
 .../src/main/java/org/apache/kylin/dict/lookup/LookupTable.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e46d699e/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index 21221f4..90ca500 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -83,7 +83,7 @@ abstract public class LookupTable {
 Array key = new Array(keyCols);
 
 if (data.containsKey(key))
-throw new IllegalStateException("Dup key found, key=" + 
toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + 
toString(value));
+throw new IllegalStateException("The table: " + 
tableDesc.getName() + " Dup key found, key=" + toString(keyCols) + ", value1=" 
+ toString(data.get(key)) + ", value2=" + toString(value));
 
 data.put(key, value);
 }

[19/50] [abbrv] kylin git commit: KYLIN-2337 minor improvement, still give good error info

[lidong]

KYLIN-2337 minor improvement, still give good error info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/122b15a2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/122b15a2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/122b15a2

Branch: refs/heads/master-hbase1.x
Commit: 122b15a2a5fb7385e64839edfb9cbdaec5fde841
Parents: 7509bee
Author: Yang Li 
Authored: Sat Jan 7 09:38:06 2017 +0800
Committer: Yang Li 
Committed: Sat Jan 7 09:38:06 2017 +0800

--
 .../kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/122b15a2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
index 6a7e713..c5c95a6 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
@@ -126,7 +126,8 @@ public class SortedIteratorMergerWithLimit extends SortedIt
 
 //TODO: remove this check when validated
 if (last != null) {
-Preconditions.checkState(comparator.compare(last, fetched) <= 
0, "Not sorted!");
+if (comparator.compare(last, fetched) > 0)
+throw new IllegalStateException("Not sorted! last: " + 
last + " fetched: " + fetched);
 }
 
 last = fetched;

[07/50] [abbrv] kylin git commit: KYLIN-2360 fix potential bugs reported by Sonar

[lidong]

KYLIN-2360 fix potential bugs reported by Sonar


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c64f55b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c64f55b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c64f55b2

Branch: refs/heads/master-hbase1.x
Commit: c64f55b2e34089d10c8a55301b62f96aa597d8f1
Parents: 5683954
Author: Billy Liu 
Authored: Thu Jan 5 17:41:06 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 5 17:41:06 2017 +0800

--
 .../adapter/enumerable/EnumerableWindow.java| 48 
 .../apache/kylin/common/KylinConfigBase.java| 11 +++--
 .../common/util/MemoryBudgetController.java |  4 +-
 .../apache/kylin/common/util/ThreadUtil.java| 47 ---
 .../org/apache/kylin/common/util/TimeUtil.java  |  2 +-
 .../apache/kylin/common/util/ZipFileUtils.java  |  5 +-
 .../kylin/cube/cli/DumpDictionaryCLI.java   |  6 ++-
 .../cube/inmemcubing/DoggedCubeBuilder.java |  3 ++
 .../cube/inmemcubing/InMemCubeBuilder.java  |  2 +
 .../kylin/cube/inmemcubing/MemDiskStore.java|  1 +
 .../kylin/cube/model/AggregationGroup.java  |  3 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |  3 +-
 .../apache/kylin/cube/model/DictionaryDesc.java |  4 +-
 .../apache/kylin/cube/model/DimensionDesc.java  |  3 +-
 .../kylin/cube/model/HBaseMappingDesc.java  |  3 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java |  3 +-
 .../kylin/cube/model/v1_4_0/CubeDesc.java   |  3 +-
 .../kylin/gridtable/GTAggregateScanner.java |  4 +-
 .../org/apache/kylin/dict/CachedTreeMap.java| 16 +++
 .../org/apache/kylin/dict/TrieDictionary.java   |  2 +-
 .../apache/kylin/dict/TrieDictionaryForest.java |  2 +-
 .../apache/kylin/dimension/BooleanDimEnc.java   |  2 +-
 .../kylin/measure/MeasureTypeFactory.java   |  3 +-
 .../kylin/metadata/model/FunctionDesc.java  |  8 ++--
 .../kylin/metadata/model/JoinTableDesc.java |  4 +-
 .../kylin/metadata/model/MeasureDesc.java   |  3 +-
 .../metadata/model/ModelDimensionDesc.java  |  9 ++--
 .../kylin/metadata/model/PartitionDesc.java |  4 +-
 .../apache/kylin/metadata/model/TableDesc.java  |  2 +-
 .../kylin/metadata/model/TableExtDesc.java  |  3 +-
 .../apache/kylin/metadata/model/TableRef.java   |  3 +-
 .../metadata/project/RealizationEntry.java  |  4 +-
 .../org/apache/kylin/metadata/tuple/Tuple.java  |  2 +-
 .../apache/kylin/metadata/tuple/TupleInfo.java  | 10 ++--
 .../org/apache/kylin/source/ReadableTable.java  |  3 +-
 .../kylin/source/datagen/ColumnGenerator.java   |  2 +-
 .../kylin/engine/mr/DFSFileTableReader.java |  4 +-
 .../engine/mr/common/HadoopStatusGetter.java|  2 +
 .../engine/mr/common/MapReduceExecutable.java   |  3 +-
 .../mr/steps/MapContextGTRecordWriter.java  |  1 +
 .../engine/mr/steps/SelfDefineSortableKey.java  |  2 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |  4 ++
 .../engine/spark/cube/BufferedCuboidWriter.java |  3 ++
 server-base/pom.xml |  7 ---
 .../kylin/rest/request/PrepareSqlRequest.java   |  3 +-
 .../kylin/rest/service/BadQueryDetector.java|  5 +-
 .../apache/kylin/rest/service/CubeService.java  |  4 ++
 .../kylin/rest/service/DiagnosisService.java|  6 ++-
 .../cardinality/ColumnCardinalityMapper.java|  3 +-
 source-kafka/pom.xml|  1 -
 .../kylin/source/kafka/config/BrokerConfig.java |  4 +-
 .../common/coprocessor/AggregationCache.java|  4 +-
 .../hbase/cube/v2/ExpectedSizeIterator.java |  2 +
 .../storage/hbase/util/HbaseStreamingInput.java |  2 +
 .../kylin/storage/hbase/util/PingHBaseCLI.java  |  1 +
 .../hbase/util/ZookeeperDistributedJobLock.java |  2 +
 .../org/apache/kylin/tool/CubeMetaIngester.java |  3 +-
 .../org/apache/kylin/tool/DiagnosisInfoCLI.java |  1 +
 .../apache/kylin/tool/HBaseUsageExtractor.java  |  2 +-
 59 files changed, 137 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
 
b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
index 203ce02..547210c 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
@@ -113,54 +113,6 @@ public class EnumerableWindow extends Window implements 
EnumerableRel {
 }
 }
 
-private void sampleOfTheGeneratedWindowedAggregate() {
-// Here's 

[30/50] [abbrv] kylin git commit: KYLIN-2371 Allow overwrite default spark conf at cube level

[lidong]

KYLIN-2371 Allow overwrite default spark conf at cube level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ee74a74e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ee74a74e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ee74a74e

Branch: refs/heads/master-hbase1.x
Commit: ee74a74e4d17a45b3832912381220af39e71f3d5
Parents: 66bca9a
Author: shaofengshi 
Authored: Mon Jan 9 15:23:22 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 build/conf/kylin-spark-conf.properties  |  5 +--
 build/conf/kylin.properties |  6 ++--
 .../apache/kylin/common/KylinConfigBase.java| 12 ---
 .../kylin/job/constant/ExecutableConstants.java |  1 +
 .../spark/SparkBatchCubingJobBuilder2.java  |  2 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |  7 ++--
 .../kylin/engine/spark/SparkExecutable.java | 38 +---
 .../test_case_data/sandbox/kylin.properties |  3 +-
 8 files changed, 55 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/build/conf/kylin-spark-conf.properties
--
diff --git a/build/conf/kylin-spark-conf.properties 
b/build/conf/kylin-spark-conf.properties
index 5e6dafe..143e7e4 100644
--- a/build/conf/kylin-spark-conf.properties
+++ b/build/conf/kylin-spark-conf.properties
@@ -1,5 +1,5 @@
 spark.yarn.submit.file.replication=1
-spark.yarn.executor.memoryOverhead=200
+spark.yarn.executor.memoryOverhead=1024
 spark.yarn.driver.memoryOverhead=384
 spark.master=yarn
 spark.submit.deployMode=cluster
@@ -20,7 +20,8 @@ spark.executor.cores=4
 spark.executor.instances=8
 spark.history.kerberos.keytab=none
 spark.history.kerberos.principal=none
-#spark.yarn.jar=hdfs://namenode:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 spark.driver.extraJavaOptions=-Dhdp.version=current
 spark.yarn.am.extraJavaOptions=-Dhdp.version=current
 spark.executor.extraJavaOptions=-Dhdp.version=current
+#spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+#spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec

http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 54430f0..6efa423 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -131,16 +131,16 @@ kylin.engine.mr.mapper-input-rows=100
 ### Spark Engine ###
 
 # Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run 
spark-submit
-kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+#kylin.engine.spark.env.hadoop-conf-dir=/etc/hive/conf
 
 # Spark job submission properties file, default be 
$KYLIN_HOME/conf/kylin-spark-conf.properties
 #kylin.engine.spark.properties-file=
 
 # Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=50
+kylin.engine.spark.rdd-partition-cut-mb=10
 
 # Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=10
+kylin.engine.spark.min-partition=1
 
 # Max partition numbers of rdd
 kylin.engine.spark.max-partition=5000

http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7d6ac2b..36ddbf4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -695,6 +695,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.engine.mr.config-override.");
 }
 
+public Map getSparkConfigOverride() {
+return getPropertiesByPrefix("kylin.engine.spark.config-override.");
+}
+
 public double getDefaultHadoopJobReducerInputMB() {
 return 
Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500"));
 }
@@ -739,8 +743,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 // ENGINE.SPARK
 // 

 
-public String getSparkHadoopConfDir() {
-return getRequired("kylin.engine.spark.env.hadoop-conf-dir");
+public String getHadoopConfDir() {
+return getOptional("kylin.engine.spark.env.hadoop-conf-dir", "");
 }
 
 public String getSparkConfFile() {
@@ 

[23/50] [abbrv] kylin git commit: KYLIN-2353 Serialize BitmapCounter with distinct count

[lidong]

KYLIN-2353 Serialize BitmapCounter with distinct count

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/731a53a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/731a53a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/731a53a5

Branch: refs/heads/master-hbase1.x
Commit: 731a53a51a32fb1314b4e1282f158e3b7ed819a5
Parents: aa57446
Author: kangkaisen 
Authored: Sat Dec 31 20:28:30 2016 +0800
Committer: Yang Li 
Committed: Sat Jan 7 19:08:37 2017 +0800

--
 .../kylin/measure/bitmap/BitmapCounter.java | 105 ---
 .../bitmap/BitmapDistinctCountAggFunc.java  |   2 +-
 .../kylin/measure/bitmap/BitmapSerializer.java  |  15 +--
 3 files changed, 70 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/731a53a5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index aeb14ba..a18ac4a 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -35,6 +35,8 @@ public class BitmapCounter implements 
Comparable {
 
 private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
 private final int VERSION = 2;
+private Integer count;
+private ByteBuffer buffer;
 
 public BitmapCounter() {
 }
@@ -43,18 +45,41 @@ public class BitmapCounter implements 
Comparable {
 merge(another);
 }
 
+private MutableRoaringBitmap getBitmap() {
+if (!bitmap.isEmpty()) {
+return bitmap;
+}
+
+if (buffer != null) {
+int version = buffer.getInt();
+int size = buffer.getInt();
+count = buffer.getInt();
+
+try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(buffer))) {
+bitmap.deserialize(is);
+} catch (IOException e) {
+throw new RuntimeException("deserialize bitmap failed!");
+}
+
+buffer = null;
+}
+
+return bitmap;
+}
+
 public void clear() {
-bitmap.clear();
+getBitmap().clear();
 }
 
 public BitmapCounter clone() {
 BitmapCounter newCounter = new BitmapCounter();
-newCounter.bitmap = bitmap.clone();
+newCounter.bitmap = getBitmap().clone();
 return newCounter;
 }
 
 public void add(int value) {
-bitmap.add(value);
+getBitmap().add(value);
+count = getBitmap().getCardinality();
 }
 
 public void add(byte[] value) {
@@ -77,34 +102,43 @@ public class BitmapCounter implements 
Comparable {
 }
 
 public void merge(BitmapCounter another) {
-this.bitmap.or(another.bitmap);
+getBitmap().or(another.getBitmap());
+count = getBitmap().getCardinality();
 }
 
 public void intersect(BitmapCounter another) {
-this.bitmap.and(another.bitmap);
+getBitmap().and(another.getBitmap());
+count = getBitmap().getCardinality();
 }
 
-public long getCount() {
-return this.bitmap.getCardinality();
+public int getCount() {
+if (count != null) {
+return count;
+}
+
+return getBitmap().getCardinality();
 }
 
 public int getMemBytes() {
-return this.bitmap.getSizeInBytes();
+return getBitmap().getSizeInBytes();
 }
 
 public Iterator iterator() {
-return bitmap.iterator();
+return getBitmap().iterator();
 }
 
 public void writeRegisters(ByteBuffer out) throws IOException {
 ByteArrayOutputStream bos = new ByteArrayOutputStream();
 DataOutputStream dos = new DataOutputStream(bos);
+MutableRoaringBitmap bitmap = getBitmap();
 bitmap.runOptimize();
 bitmap.serialize(dos);
 dos.close();
 ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray());
+
 out.putInt(VERSION);
-out.putInt(bos.size() + 4 + 4);
+out.putInt(bos.size() + 4 + 4 + 4);
+out.putInt(getCount());
 out.put(bb);
 }
 
@@ -116,13 +150,30 @@ public class BitmapCounter implements 
Comparable {
 if (version == VERSION) {
 @SuppressWarnings("unused")
 int size = in.getInt();
+count = in.getInt();
+in.position(mark);
+buffer = cloneBuffer(in, size);
 } else {
 in.position(mark);
+ 

[09/50] [abbrv] kylin git commit: KYLIN-2360 fix minor SonarCube issue

[lidong]

KYLIN-2360 fix minor SonarCube issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/503d232d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/503d232d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/503d232d

Branch: refs/heads/master-hbase1.x
Commit: 503d232dc1875d504959a3e00da2eb62a7b7d599
Parents: 980b03b
Author: shaofengshi 
Authored: Thu Jan 5 18:04:56 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 5 18:09:45 2017 +0800

--
 .../adapter/enumerable/EnumerableWindow.java| 53 +++-
 .../kylin/common/persistence/ResourceTool.java  |  5 +-
 .../apache/kylin/common/util/StringUtil.java|  2 +-
 .../org/apache/kylin/dict/CacheDictionary.java  | 29 ++-
 4 files changed, 71 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/503d232d/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
 
b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
index 547210c..216b07c 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
@@ -80,6 +80,7 @@ public class EnumerableWindow extends Window implements 
EnumerableRel {
 constants, rowType, groups);
 }
 
+@Override
 public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery 
mq) {
 return super.computeSelfCost(planner, mq)
 .multiplyBy(EnumerableConvention.COST_MULTIPLIER);
@@ -104,6 +105,7 @@ public class EnumerableWindow extends Window implements 
EnumerableRel {
 this.constants = constants;
 }
 
+@Override
 public Expression field(BlockBuilder list, int index, Type 
storageType) {
 if (index < actualInputFieldCount) {
 Expression current = list.append("current", row);
@@ -113,6 +115,55 @@ public class EnumerableWindow extends Window implements 
EnumerableRel {
 }
 }
 
+private void sampleOfTheGeneratedWindowedAggregate() {
+// Here's overview of the generated code
+// For each list of rows that have the same partitioning key, evaluate
+// all of the windowed aggregate functions.
+
+// builder
+Iterator iterator = null;
+
+// builder3
+Integer[] rows = iterator.next();
+
+int prevStart = -1;
+int prevEnd = -1;
+
+for (int i = 0; i < rows.length; i++) {
+// builder4
+Integer row = rows[i];
+
+int start = 0;
+int end = 100;
+if (start != prevStart || end != prevEnd) {
+// builder5
+int actualStart = 0;
+if (start != prevStart || end < prevEnd) {
+// builder6
+// recompute
+actualStart = start;
+// implementReset
+} else { // must be start == prevStart && end > prevEnd
+actualStart = prevEnd + 1;
+}
+prevStart = start;
+prevEnd = end;
+
+if (start != -1) {
+for (int j = actualStart; j <= end; j++) {
+// builder7
+// implementAdd
+}
+}
+// implementResult
+// list.add(new Xxx(row.deptno, row.empid, sum, count));
+}
+}
+// multiMap.clear(); // allows gc
+// source = Linq4j.asEnumerable(list);
+}
+
+@Override
 public Result implement(EnumerableRelImplementor implementor, Prefer pref) 
{
 final JavaTypeFactory typeFactory = implementor.getTypeFactory();
 final EnumerableRel child = (EnumerableRel) getInput();
@@ -121,7 +172,7 @@ public class EnumerableWindow extends Window implements 
EnumerableRel {
 Expression source_ = builder.append("source", result.block);
 
 final List translatedConstants =
-new ArrayList(constants.size());
+new ArrayList<>(constants.size());
 for (RexLiteral constant : constants) {
 translatedConstants.add(
 RexToLixTranslator.translateLiteral(constant, 
constant.getType(),

http://git-wip-us.apache.org/repos/asf/kylin/blob/503d232d/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java

[05/50] [abbrv] kylin git commit: minor, set reducer number in job

[lidong]

minor, set reducer number in job

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b736175f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b736175f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b736175f

Branch: refs/heads/master-hbase1.x
Commit: b736175f417f48fe07d01c0e2c99e040c3242d6f
Parents: df5faf3
Author: Roger Shi 
Authored: Thu Jan 5 15:38:51 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 5 15:51:01 2017 +0800

--
 .../java/org/apache/kylin/engine/mr/steps/CuboidJob.java | 2 +-
 .../apache/kylin/engine/mr/steps/LayerReducerNumSizing.java  | 8 ++--
 .../org/apache/kylin/engine/mr/steps/MergeCuboidJob.java | 2 +-
 3 files changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b736175f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index ef25b55..b2e186d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -131,7 +131,7 @@ public class CuboidJob extends AbstractHadoopJob {
 // add metadata to distributed cache
 attachSegmentMetadataWithDict(segment, job.getConfiguration());
 
-LayerReducerNumSizing.setReduceTaskNum(job, segment, 
getTotalMapInputMB(), nCuboidLevel);
+
job.setNumReduceTasks(LayerReducerNumSizing.getReduceTaskNum(segment, 
getTotalMapInputMB(), nCuboidLevel));
 
 this.deletePath(job.getConfiguration(), output);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b736175f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
index 713a95c..7ce9842 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
@@ -20,8 +20,6 @@ package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
 
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.model.CubeDesc;
@@ -34,7 +32,7 @@ public class LayerReducerNumSizing {
 
 private static final Logger logger = 
LoggerFactory.getLogger(LayerReducerNumSizing.class);
 
-public static void setReduceTaskNum(Job job, CubeSegment cubeSegment, 
double totalMapInputMB, int level) throws ClassNotFoundException, IOException, 
InterruptedException, JobException {
+public static int getReduceTaskNum(CubeSegment cubeSegment, double 
totalMapInputMB, int level) throws ClassNotFoundException, IOException, 
InterruptedException, JobException {
 CubeDesc cubeDesc = cubeSegment.getCubeDesc();
 KylinConfig kylinConfig = cubeDesc.getConfig();
 
@@ -76,9 +74,7 @@ public class LayerReducerNumSizing {
 // no more than 500 reducer by default
 numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), 
numReduceTasks);
 
-job.setNumReduceTasks(numReduceTasks);
-
-logger.info("Setting " + Reducer.Context.NUM_REDUCES + "=" + 
numReduceTasks);
+return numReduceTasks;
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b736175f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
index d9ff616..84b76e3 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
@@ -82,7 +82,7 @@ public class MergeCuboidJob extends CuboidJob {
 // TODO actually only dictionaries from merging segments are needed
 attachCubeMetadataWithDict(cube, job.getConfiguration());
 
-LayerReducerNumSizing.setReduceTaskNum(job, 
cube.getSegmentById(segmentID), getTotalMapInputMB(), -1);
+

[39/50] [abbrv] kylin git commit: minor, move duplicated ByteBufferBackedInputStream to common package

[lidong]

minor, move duplicated ByteBufferBackedInputStream to common package


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0aadb12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0aadb12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0aadb12

Branch: refs/heads/master-hbase1.x
Commit: b0aadb12d1e25005445eda0d912c63a941ad929a
Parents: 3a36c66
Author: gaodayue 
Authored: Tue Jan 10 11:51:38 2017 +0800
Committer: gaodayue 
Committed: Tue Jan 10 11:52:04 2017 +0800

--
 .../util/ByteBufferBackedInputStream.java   | 61 
 .../kylin/measure/bitmap/BitmapCounter.java | 38 +---
 .../measure/bitmap/BitmapAggregatorTest.java|  2 +-
 .../kylin/measure/bitmap/BitmapCounterTest.java |  2 +-
 .../measure/bitmap/BitmapSerializerTest.java|  2 +-
 .../source/kafka/TimedJsonStreamParser.java |  2 +-
 .../kafka/util/ByteBufferBackedInputStream.java | 52 -
 7 files changed, 66 insertions(+), 93 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
 
b/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
new file mode 100644
index 000..7332ff2
--- /dev/null
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+/**
+ * Utility to view content available in a {@link ByteBuffer} as a {@link 
InputStream}.
+ *
+ * Not thread-safe
+ */
+public class ByteBufferBackedInputStream extends InputStream {
+private final ByteBuffer buffer;
+
+public ByteBufferBackedInputStream(ByteBuffer buffer) {
+this.buffer = buffer;
+}
+
+@Override
+public int available() throws IOException {
+return buffer.remaining();
+}
+
+@Override
+public int read() throws IOException {
+return buffer.hasRemaining() ? (buffer.get() & 0xFF) : -1;
+}
+
+@Override
+public int read(byte[] b, int off, int len) throws IOException {
+if (len == 0) {
+return 0;
+}
+
+if (!buffer.hasRemaining()) {
+return -1;
+}
+
+len = Math.min(buffer.remaining(), len);
+buffer.get(b, off, len);
+return len;
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index 43ee506..19d7f5d 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -22,10 +22,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 
+import org.apache.kylin.common.util.ByteBufferBackedInputStream;
 import org.roaringbitmap.buffer.MutableRoaringBitmap;
 
 /**
@@ -84,18 +84,6 @@ public class BitmapCounter implements 
Comparable, java.io.Seriali
 count = null;
 }
 
-public void add(byte[] value) {
-add(value, 0, value.length);
-}
-
-public void add(byte[] value, int offset, int length) {
-if (value == null || length == 0) {
-return;
-}
-
-

[03/50] [abbrv] kylin git commit: KYLIN-2358 CuboidReducer has too many if (aggrMask[i]) checks

[lidong]

KYLIN-2358 CuboidReducer has too many if (aggrMask[i]) checks


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b5257a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b5257a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b5257a1

Branch: refs/heads/master-hbase1.x
Commit: 4b5257a1dc4b1cf90804e5273f27944de30b8fbf
Parents: 9c98574
Author: shaofengshi 
Authored: Thu Jan 5 13:49:08 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 5 13:49:08 2017 +0800

--
 .../kylin/measure/MeasureAggregators.java   |  9 +++
 .../kylin/engine/mr/steps/CuboidReducer.java| 26 
 2 files changed, 19 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4b5257a1/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
index 2b28302..8951b69 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
@@ -77,13 +77,10 @@ public class MeasureAggregators implements Serializable {
 }
 }
 
-public void aggregate(Object[] values, boolean[] aggrMask) {
+public void aggregate(Object[] values, int[] measures) {
 assert values.length == descLength;
-assert aggrMask.length == descLength;
-
-for (int i = 0; i < descLength; i++) {
-if (aggrMask[i])
-aggs[i].aggregate(values[i]);
+for (int i = 0; i < measures.length; i++) {
+aggs[measures[i]].aggregate(values[measures[i]]);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/4b5257a1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index afd29e3..495be77 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
 
+import com.google.common.collect.Lists;
 import org.apache.hadoop.io.Text;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.CubeManager;
@@ -51,7 +52,7 @@ public class CuboidReducer extends KylinReducer {
 private MeasureAggregators aggs;
 
 private int cuboidLevel;
-private boolean[] needAggr;
+private int[] needAggrMeasures;
 private Object[] input;
 private Object[] result;
 private int vcounter;
@@ -76,13 +77,22 @@ public class CuboidReducer extends KylinReducer {
 
 input = new Object[measuresDescs.size()];
 result = new Object[measuresDescs.size()];
-needAggr = new boolean[measuresDescs.size()];
 
-if (cuboidLevel > 0) {
-for (int i = 0; i < measuresDescs.size(); i++) {
-needAggr[i] = 
!measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid();
+List needAggMeasuresList = Lists.newArrayList();
+for (int i = 0; i < measuresDescs.size(); i++) {
+if (cuboidLevel == 0) {
+needAggMeasuresList.add(i);
+} else {
+if 
(!measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid()) {
+needAggMeasuresList.add(i);
+}
 }
 }
+
+needAggrMeasures = new int[needAggMeasuresList.size()];
+for (int i = 0; i < needAggMeasuresList.size(); i++) {
+needAggrMeasures[i] = needAggMeasuresList.get(i);
+}
 }
 
 @Override
@@ -94,11 +104,7 @@ public class CuboidReducer extends KylinReducer {
 logger.info("Handling value with ordinal (This is not KV 
number!): " + vcounter);
 }
 codec.decode(ByteBuffer.wrap(value.getBytes(), 0, 
value.getLength()), input);
-if (cuboidLevel > 0) {
-aggs.aggregate(input, needAggr);
-} else {
-aggs.aggregate(input);
-}
+aggs.aggregate(input, needAggrMeasures);
 }
 aggs.collectStates(result);
 

[31/50] [abbrv] kylin git commit: modify default partition setting

[lidong]

modify default partition setting


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2cf52b49
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2cf52b49
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2cf52b49

Branch: refs/heads/master-hbase1.x
Commit: 2cf52b49ed9a1158cb2ec0ded6d907f90ee024fe
Parents: b9a69ef
Author: shaofengshi 
Authored: Thu Jan 5 19:56:10 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 build/conf/kylin.properties | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2cf52b49/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index bd0bbd4..54430f0 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -137,13 +137,13 @@ kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
 #kylin.engine.spark.properties-file=
 
 # Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=200
+kylin.engine.spark.rdd-partition-cut-mb=50
 
 # Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=1
+kylin.engine.spark.min-partition=10
 
 # Max partition numbers of rdd
-kylin.engine.spark.max-partition=500
+kylin.engine.spark.max-partition=5000
 
 ### CUBE | DICTIONARY ###
 

[36/50] [abbrv] kylin git commit: KYLIN-2331 use org.apache.htrace.Trace to find htrace-core.jar

[lidong]

KYLIN-2331 use org.apache.htrace.Trace to find htrace-core.jar


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b9a69efa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b9a69efa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b9a69efa

Branch: refs/heads/master-hbase1.x
Commit: b9a69efaea27d915ae44f75db436b6c9b6b7a4ec
Parents: e864cd3
Author: shaofengshi 
Authored: Tue Jan 3 13:36:49 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 16:58:11 2017 +0800

--
 .../apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a69efa/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index c5d47e7..9431468 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -59,10 +59,12 @@ public class SparkBatchCubingJobBuilder2 extends 
BatchCubingJobBuilder2 {
 StringBuilder jars = new StringBuilder();
 
 StringUtil.appendWithSeparator(jars, 
findJar("org.htrace.HTraceConfiguration")); // htrace-core.jar
-StringUtil.appendWithSeparator(jars, 
findJar("org.cloudera.htrace.HTraceConfiguration"));
+StringUtil.appendWithSeparator(jars, 
findJar("org.apache.htrace.Trace")); // htrace-core.jar
+StringUtil.appendWithSeparator(jars, 
findJar("org.cloudera.htrace.HTraceConfiguration")); // htrace-core.jar
 StringUtil.appendWithSeparator(jars, 
findJar("org.apache.hadoop.hbase.client.HConnection")); // hbase-client.jar
 StringUtil.appendWithSeparator(jars, 
findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar
 StringUtil.appendWithSeparator(jars, 
findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar
+StringUtil.appendWithSeparator(jars, 
findJar("com.yammer.metrics.core.Gauge")); // metrics-core.jar
 
 StringUtil.appendWithSeparator(jars, 
seg.getConfig().getSparkAdditionalJars());
 sparkExecutable.setJars(jars.toString());

[06/50] [abbrv] kylin git commit: KYLIN-2333 fix ArrayIndexOutOfBoundsException in JobBuilderSupport

[lidong]

KYLIN-2333 fix ArrayIndexOutOfBoundsException in JobBuilderSupport


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5683954b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5683954b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5683954b

Branch: refs/heads/master-hbase1.x
Commit: 5683954bafec3fb02cc2ccda56387dacceb6b593
Parents: b736175
Author: shaofengshi 
Authored: Thu Jan 5 16:32:28 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 5 16:32:28 2017 +0800

--
 .../main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5683954b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 122b8ba..86ac880 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -193,7 +193,7 @@ public class JobBuilderSupport {
 }
 
 public String[] getCuboidOutputPaths(String cuboidRootPath, int levels) {
-String[] paths = new String[levels];
+String[] paths = new String[levels + 1];
 for (int i = 0; i <= levels; i++) {
 if (i == 0) {
 paths[i] = cuboidRootPath + "base_cuboid";

[13/50] [abbrv] kylin git commit: KYLIN-2287 fix the button display issue with js

[lidong]

KYLIN-2287 fix the button display issue with js

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49de556f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49de556f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49de556f

Branch: refs/heads/master-hbase1.x
Commit: 49de556f00ff8fd854654daf1bc7e1d26c1fb879
Parents: f267ba9
Author: ShaoChinYu 
Authored: Fri Jan 6 12:22:59 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 6 12:29:35 2017 +0800

--
 webapp/app/js/controllers/models.js | 36 
 webapp/app/partials/models/models_tree.html |  6 ++--
 2 files changed, 39 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/49de556f/webapp/app/js/controllers/models.js
--
diff --git a/webapp/app/js/controllers/models.js 
b/webapp/app/js/controllers/models.js
index e3ee038..fd21845 100644
--- a/webapp/app/js/controllers/models.js
+++ b/webapp/app/js/controllers/models.js
@@ -183,7 +183,41 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, 
$routeParams, $location,
 });
   };
 
+  function changePositionOfScrollBar(){
+//get which button be clicked
+var btn = window.event.srcElement || window.event.target;
+//get current position of scroll bar
+var scrollTop =$("#cube_model_trees").scrollTop();
+//get total length of scroll bar
+var scrollHeight  = 
document.getElementById('cube_model_trees').scrollHeight;
+//get the position of clicked button relative to the top of window
+var offsetTop  =$(btn).offset().top;
+//get the position of the container relative to the top of window
+var treeOffsetTop = $("#cube_model_trees").offset().top;
+
+//distance from button to the top of tree model container
+var minor = offsetTop - treeOffsetTop;
+//height of tree model container
+var  viewH =$("#cube_model_trees").height();
+
+//change scroll bar to show the dropdown menu
+if(minor + 100 > viewH){//100 is the height of dropdowm menu
+  if((scrollHeight - scrollTop - viewH)>=minor+100-viewH){
+
document.getElementById('cube_model_trees').scrollTop+=(minor+120-viewH);
+  }else{
+var node=document.createElement("LI");
+node.style.height = (minor+120-viewH)+"px";
+document.getElementById("models-tree").appendChild(node);
+var  viewH =$("#cube_model_trees").height();//可见高度
+
document.getElementById('cube_model_trees').scrollTop+=(minor+120-viewH);
+
+  }
+}
+  }
+
   $scope.listModelAccess = function (model) {
+changePositionOfScrollBar();
+
 if(model.uuid){
   AccessService.list({type: "DataModelDesc", uuid: model.uuid}, function 
(accessEntities) {
 model.accessEntities = accessEntities;
@@ -196,6 +230,8 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, 
$routeParams, $location,
 }
   })
 }
+
+
   };
 
   var ModelDetailModalCtrl = function ($scope, $location, $modalInstance, 
scope) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/49de556f/webapp/app/partials/models/models_tree.html
--
diff --git a/webapp/app/partials/models/models_tree.html 
b/webapp/app/partials/models/models_tree.html
index 944d2dc..eba7d29 100644
--- a/webapp/app/partials/models/models_tree.html
+++ b/webapp/app/partials/models/models_tree.html
@@ -41,10 +41,10 @@
   
 Models
   
+
+
 
-
-
-
+
   
 
 

[22/50] [abbrv] kylin git commit: KYLIN-2349 Serialize BitmapCounter with peekLength

[lidong]

KYLIN-2349 Serialize BitmapCounter with peekLength

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa574462
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa574462
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa574462

Branch: refs/heads/master-hbase1.x
Commit: aa574462aad0620fe10343613dc4c8d761c0d6a0
Parents: 746e380
Author: kangkaisen 
Authored: Sat Dec 31 15:41:07 2016 +0800
Committer: Yang Li 
Committed: Sat Jan 7 19:08:15 2017 +0800

--
 .../kylin/measure/bitmap/BitmapCounter.java | 53 ++--
 1 file changed, 37 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aa574462/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index a7f277e..aeb14ba 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -34,6 +34,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
 public class BitmapCounter implements Comparable {
 
 private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+private final int VERSION = 2;
 
 public BitmapCounter() {
 }
@@ -102,15 +103,51 @@ public class BitmapCounter implements 
Comparable {
 bitmap.serialize(dos);
 dos.close();
 ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray());
+out.putInt(VERSION);
+out.putInt(bos.size() + 4 + 4);
 out.put(bb);
 }
 
 public void readRegisters(ByteBuffer in) throws IOException {
+int mark = in.position();
+int version = in.getInt();
+
+// keep forward compatibility
+if (version == VERSION) {
+@SuppressWarnings("unused")
+int size = in.getInt();
+} else {
+in.position(mark);
+}
+
 try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(in))) {
 bitmap.deserialize(is);
 }
 }
 
+public int peekLength(ByteBuffer in) {
+int mark = in.position();
+int len;
+int version = in.getInt();
+
+// keep forward compatibility
+if (version == VERSION) {
+len = in.getInt() ;
+} else {
+in.position(mark);
+try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(in))) {
+MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+bitmap.deserialize(is);
+len = in.position() - mark;
+} catch (IOException e) {
+throw new IllegalStateException(e);
+}
+}
+
+in.position(mark);
+return len;
+}
+
 @Override
 public String toString() {
 long count = getCount();
@@ -169,22 +206,6 @@ public class BitmapCounter implements 
Comparable {
 return -1;
 }
 
-public int peekLength(ByteBuffer in) {
-int mark = in.position();
-int len;
-
-MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
-try (DataInputStream is = new DataInputStream(new 
ByteBufferBackedInputStream(in))) {
-bitmap.deserialize(is);
-} catch (IOException e) {
-throw new IllegalStateException(e);
-}
-
-len = in.position() - mark;
-in.position(mark);
-return len;
-}
-
 private class ByteBufferBackedInputStream extends InputStream {
 private final ByteBuffer buffer;
 

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cb6ce7d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cb6ce7d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cb6ce7d6

Branch: refs/heads/yang22-hbase102
Commit: cb6ce7d6b57cfa2fefe6dc944691abfbab3e799a
Parents: 8bfef64
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jan 12 11:31:26 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cb6ce7d6/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cb6ce7d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cb6ce7d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cb6ce7d6

Branch: refs/heads/yang22-cdh5.7
Commit: cb6ce7d6b57cfa2fefe6dc944691abfbab3e799a
Parents: 8bfef64
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jan 12 11:31:26 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cb6ce7d6/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c208881
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c208881
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c208881

Branch: refs/heads/yang22-cdh5.7
Commit: 5c20888179e576d66edf05f1b17493d931d823d8
Parents: cb6ce7d
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Thu Jan 12 11:33:06 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5c208881/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/5c208881/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5c208881/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 8fed4db68 -> 5c2088817 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/cb6ce7d6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/cb6ce7d6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cb6ce7d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cb6ce7d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cb6ce7d6

Branch: refs/heads/yang22-hbase1.x
Commit: cb6ce7d6b57cfa2fefe6dc944691abfbab3e799a
Parents: 8bfef64
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jan 12 11:31:26 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cb6ce7d6/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 8d5a28836 -> cb6ce7d6b (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/cb6ce7d6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/cb6ce7d6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 

[3/3] kylin git commit: KYLIN-2233 Support HBase 1.0.2

[billyliu]

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/80d20b4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/80d20b4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/80d20b4b

Branch: refs/heads/yang22-hbase102
Commit: 80d20b4b828d4c24f608309dc2a5a38a14b2e684
Parents: 8d5a288
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: Billy Liu 
Committed: Thu Jan 12 10:29:38 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/80d20b4b/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/80d20b4b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/80d20b4b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of 

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 aba62e894 -> 80d20b4b8 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/8d5a2883/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8d5a2883/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 ae56a4e57 -> 8fed4db68 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/8d5a2883/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8d5a2883/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 

[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[billyliu]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8fed4db6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8fed4db6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8fed4db6

Branch: refs/heads/yang22-cdh5.7
Commit: 8fed4db68510a51ee8520e95db40fc2b73eb2051
Parents: 8d5a288
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Billy Liu 
Committed: Thu Jan 12 10:27:14 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8fed4db6/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/8fed4db6/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8fed4db6/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8d5a2883
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8d5a2883
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8d5a2883

Branch: refs/heads/yang22-hbase1.x
Commit: 8d5a28836e220cae139f5da102e623e0b3f327b1
Parents: 8bfef64
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Thu Jan 12 10:24:09 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8d5a2883/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x ac9d84450 -> 8d5a28836 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/8d5a2883/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8d5a2883/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 

[4/4] kylin git commit: KYLIN-2233 Support HBase 1.0.2

[billyliu]

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aba62e89
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aba62e89
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aba62e89

Branch: refs/heads/yang22-hbase102
Commit: aba62e894f2a3705e4b88926d7428873370d490d
Parents: ac9d844
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: Billy Liu 
Committed: Thu Jan 12 10:05:10 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aba62e89/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/aba62e89/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/aba62e89/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of 

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac9d8445
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac9d8445
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac9d8445

Branch: refs/heads/yang22-hbase102
Commit: ac9d844503553125042068435a77c0119ca138fa
Parents: 8bfef64
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Thu Jan 12 10:00:40 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac9d8445/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac9d8445/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac9d8445/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[1/4] kylin git commit: KYLIN-2382 fix column order for select * [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 9c093f78a -> aba62e894 (forced update)


KYLIN-2382 fix column order for select *


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8bfef648
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8bfef648
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8bfef648

Branch: refs/heads/yang22-hbase102
Commit: 8bfef648956fdc56031477016fd24628eddd0bea
Parents: 2e08815
Author: Yang Li 
Authored: Thu Jan 12 08:08:29 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 12 08:15:47 2017 +0800

--
 .../main/java/org/apache/kylin/query/schema/OLAPTable.java   | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8bfef648/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java 
b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 06ffdba..b18b90d 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -19,6 +19,8 @@
 package org.apache.kylin.query.schema;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -219,6 +221,12 @@ public class OLAPTable extends AbstractQueryableTable 
implements TranslatableTab
 logger.info("To avoid overflow, upgraded {}'s type from {} to {}", 
tableColumns.get(index), tableColumns.get(index).getType(), 
tableColumns.get(index).getUpgradedType());
 }
 
+Collections.sort(tableColumns, new Comparator() {
+@Override
+public int compare(ColumnDesc o1, ColumnDesc o2) {
+return o1.getZeroBasedIndex() - o2.getZeroBasedIndex();
+}
+});
 return Lists.newArrayList(Iterables.concat(tableColumns, 
metricColumns));
 }
 

[1/4] kylin git commit: KYLIN-2382 fix column order for select * [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 1a70112ce -> ae56a4e57 (forced update)


KYLIN-2382 fix column order for select *


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8bfef648
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8bfef648
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8bfef648

Branch: refs/heads/yang22-cdh5.7
Commit: 8bfef648956fdc56031477016fd24628eddd0bea
Parents: 2e08815
Author: Yang Li 
Authored: Thu Jan 12 08:08:29 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 12 08:15:47 2017 +0800

--
 .../main/java/org/apache/kylin/query/schema/OLAPTable.java   | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8bfef648/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java 
b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 06ffdba..b18b90d 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -19,6 +19,8 @@
 package org.apache.kylin.query.schema;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -219,6 +221,12 @@ public class OLAPTable extends AbstractQueryableTable 
implements TranslatableTab
 logger.info("To avoid overflow, upgraded {}'s type from {} to {}", 
tableColumns.get(index), tableColumns.get(index).getType(), 
tableColumns.get(index).getUpgradedType());
 }
 
+Collections.sort(tableColumns, new Comparator() {
+@Override
+public int compare(ColumnDesc o1, ColumnDesc o2) {
+return o1.getZeroBasedIndex() - o2.getZeroBasedIndex();
+}
+});
 return Lists.newArrayList(Iterables.concat(tableColumns, 
metricColumns));
 }
 

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[billyliu]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae56a4e5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae56a4e5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae56a4e5

Branch: refs/heads/yang22-cdh5.7
Commit: ae56a4e578424cf4115e42b3a2bf55e82505c395
Parents: ac9d844
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Billy Liu 
Committed: Thu Jan 12 10:03:15 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ae56a4e5/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/ae56a4e5/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ae56a4e5/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac9d8445/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac9d8445/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac9d8445
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac9d8445
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac9d8445

Branch: refs/heads/yang22-cdh5.7
Commit: ac9d844503553125042068435a77c0119ca138fa
Parents: 8bfef64
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Thu Jan 12 10:00:40 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac9d8445/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac9d8445
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac9d8445
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac9d8445

Branch: refs/heads/yang22-hbase1.x
Commit: ac9d844503553125042068435a77c0119ca138fa
Parents: 8bfef64
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Thu Jan 12 10:00:40 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac9d8445/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac9d8445/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac9d8445/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[1/3] kylin git commit: KYLIN-2382 fix column order for select * [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 288a3dc3d -> ac9d84450 (forced update)


KYLIN-2382 fix column order for select *


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8bfef648
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8bfef648
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8bfef648

Branch: refs/heads/yang22-hbase1.x
Commit: 8bfef648956fdc56031477016fd24628eddd0bea
Parents: 2e08815
Author: Yang Li 
Authored: Thu Jan 12 08:08:29 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 12 08:15:47 2017 +0800

--
 .../main/java/org/apache/kylin/query/schema/OLAPTable.java   | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8bfef648/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java 
b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 06ffdba..b18b90d 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -19,6 +19,8 @@
 package org.apache.kylin.query.schema;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -219,6 +221,12 @@ public class OLAPTable extends AbstractQueryableTable 
implements TranslatableTab
 logger.info("To avoid overflow, upgraded {}'s type from {} to {}", 
tableColumns.get(index), tableColumns.get(index).getType(), 
tableColumns.get(index).getUpgradedType());
 }
 
+Collections.sort(tableColumns, new Comparator() {
+@Override
+public int compare(ColumnDesc o1, ColumnDesc o2) {
+return o1.getZeroBasedIndex() - o2.getZeroBasedIndex();
+}
+});
 return Lists.newArrayList(Iterables.concat(tableColumns, 
metricColumns));
 }
 

kylin git commit: KYLIN-2382 fix column order for select *

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22 2e0881596 -> 8bfef6489


KYLIN-2382 fix column order for select *


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8bfef648
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8bfef648
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8bfef648

Branch: refs/heads/yang22
Commit: 8bfef648956fdc56031477016fd24628eddd0bea
Parents: 2e08815
Author: Yang Li 
Authored: Thu Jan 12 08:08:29 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 12 08:15:47 2017 +0800

--
 .../main/java/org/apache/kylin/query/schema/OLAPTable.java   | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8bfef648/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java 
b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 06ffdba..b18b90d 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -19,6 +19,8 @@
 package org.apache.kylin.query.schema;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -219,6 +221,12 @@ public class OLAPTable extends AbstractQueryableTable 
implements TranslatableTab
 logger.info("To avoid overflow, upgraded {}'s type from {} to {}", 
tableColumns.get(index), tableColumns.get(index).getType(), 
tableColumns.get(index).getUpgradedType());
 }
 
+Collections.sort(tableColumns, new Comparator() {
+@Override
+public int compare(ColumnDesc o1, ColumnDesc o2) {
+return o1.getZeroBasedIndex() - o2.getZeroBasedIndex();
+}
+});
 return Lists.newArrayList(Iterables.concat(tableColumns, 
metricColumns));
 }
 

[1/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/mhbstresstest [created] 8d1491888


http://git-wip-us.apache.org/repos/asf/kylin/blob/98c7e8f0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/98c7e8f0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/98c7e8f0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/98c7e8f0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/98c7e8f0

Branch: refs/heads/mhbstresstest
Commit: 98c7e8f0933f04d408f2371a35117ffa44376961
Parents: ee04dca
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Tue Jan 3 20:42:06 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/98c7e8f0/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[4/4] kylin git commit: relax project num for single cube

[liyang]

relax project num for single cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8d149188
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8d149188
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8d149188

Branch: refs/heads/mhbstresstest
Commit: 8d1491888312edac9656f3345c862a9ab33c0dca
Parents: ee32cb1
Author: Hongbin Ma 
Authored: Wed Jan 11 23:12:00 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Jan 11 23:12:00 2017 +0800

--
 engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8d149188/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 64e9edb..35d5474 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -84,7 +84,8 @@ public class CubingJob extends DefaultChainedExecutable {
 if (projList == null || projList.size() == 0) {
 throw new RuntimeException("Cannot find the project containing the 
cube " + cube.getName() + "!!!");
 } else if (projList.size() >= 2) {
-throw new RuntimeException("Find more than one project containing 
the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! 
");
+//throw new RuntimeException("Find more than one project 
containing the cube " + cube.getName() + ". It does't meet the uniqueness 
requirement!!! ");
+logger.warn("Find more than one project containing the cube " + 
cube.getName() + ". It does't meet the uniqueness requirement!!! ");
 }
 
 CubingJob result = new CubingJob();