kylin git commit: Add UDF version() to get kylin version, Add UT

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 408f9d43d -> 637581fb7


Add UDF version() to get kylin version, Add UT

Signed-off-by: Hongbin Ma 

Update kylin properties's UDF setting

Signed-off-by: Hongbin Ma 

Add IT cases of version() and query output check.

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/637581fb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/637581fb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/637581fb

Branch: refs/heads/master
Commit: 637581fb7f9b1ae1a3abde93795755aaceea4ecf
Parents: 408f9d4
Author: Yifan Zhang 
Authored: Tue Oct 18 16:12:07 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 8 14:50:15 2016 +0800

--
 build/conf/kylin.properties |  2 ++
 .../test_case_data/sandbox/kylin.properties |  1 +
 .../apache/kylin/query/ITKylinQueryTest.java| 28 +++-
 .../org/apache/kylin/query/KylinTestBase.java   | 25 +
 .../query/sql_verifyContent/query01.sql | 21 +++
 .../sql_verifyContent/query01.sql.expected.xml  |  4 +++
 .../org/apache/kylin/query/udf/VersionUDF.java  | 27 +++
 .../apache/kylin/query/udf/VersionUDFTest.java  | 15 +++
 8 files changed, 122 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/637581fb/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 91aa5b8..e935ebf 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -135,6 +135,8 @@ kylin.table.snapshot.max_mb=300
 
 kylin.query.scan.threshold=1000
 
+kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
+
 # 3G
 kylin.query.mem.budget=3221225472
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/637581fb/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 43b0855..0efd5c9 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -104,6 +104,7 @@ kylin.hbase.region.count.max=5
 kylin.hbase.hfile.size.gb=2
 
 kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF
+kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
 
 kylin.job.controller.lock=org.apache.kylin.job.lock.MockJobLock
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/637581fb/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 520c5e6..9c1b640 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -27,6 +27,7 @@ import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinVersion;
 import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -36,7 +37,9 @@ import 
org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.storage.hbase.HBaseStorage;
 import org.dbunit.database.DatabaseConnection;
 import org.dbunit.database.IDatabaseConnection;
+import org.dbunit.dataset.ITable;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Rule;
@@ -199,11 +202,16 @@ public class ITKylinQueryTest extends KylinTestBase {
 }
 
 @Test
-public void testVerifyQuery() throws Exception {
+public void testVerifyCountQuery() throws Exception {
 verifyResultRowCount(getQueryFolderPrefix() + 
"src/test/resources/query/sql_verifyCount");
 }
 
 @Test
+public void testVerifyContentQuery() throws Exception {
+verifyResultContent(getQueryFolderPrefix() + 
"src/test/resources/query/sql_verifyContent");
+}
+
+@Test
 public void testOrderByQuery() throws Exception {
 execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_orderby", null, true);
 // FIXME
@@ -364,4 +372,22 @@ public class ITKylinQueryTest extends KylinTestBase {
 this.batchExecuteQuery(getQueryFolderPrefix() + 

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a3fd7d5e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a3fd7d5e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a3fd7d5e

Branch: refs/heads/yang21-cdh5.7
Commit: a3fd7d5e442e2a8180eecba65626bea3b7641367
Parents: 1720f1e
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 8 14:38:52 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a3fd7d5e/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-  

[1/4] kylin git commit: KYLIN-2159 Redistribution Hive Table Step always requires row_count filename as 000000_0 [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 d4df0043e -> f6e377d82 (forced update)


KYLIN-2159 Redistribution Hive Table Step always requires row_count filename as 
00_0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1720f1eb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1720f1eb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1720f1eb

Branch: refs/heads/yang21-cdh5.7
Commit: 1720f1eb8aea8bf6807b654558508b261d679bf7
Parents: d9bd93c
Author: shaofengshi 
Authored: Tue Nov 8 12:41:05 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 8 12:42:34 2016 +0800

--
 .../org/apache/kylin/source/hive/HiveMRInput.java  | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1720f1eb/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 95828a0..52f474a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -24,6 +24,7 @@ import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.Job;
@@ -310,10 +311,22 @@ public class HiveMRInput implements IMRInput {
 KylinConfig config = getCubeSpecificConfig();
 
 try {
-
 computeRowCount(config.getCliCommandExecutor());
+Path rowCountFile = null;
+Path rowCountFolder = new Path(getRowCountOutputDir());
+FileSystem fs = FileSystem.get(rowCountFolder.toUri(), 
HadoopUtil.getCurrentConfiguration());
+for (FileStatus stat : fs.listStatus(rowCountFolder)) {
+if (stat.isDirectory() == false && 
stat.getPath().getName().startsWith("")) {
+rowCountFile = stat.getPath();
+logger.debug("Finding file " + rowCountFile);
+break;
+}
+}
+
+if (rowCountFile == null) {
+return new ExecuteResult(ExecuteResult.State.ERROR, "No 
row count file found in '" + getRowCountOutputDir() + "'");
+}
 
-Path rowCountFile = new Path(getRowCountOutputDir(), 
"00_0");
 long rowCount = readRowCountFromFile(rowCountFile);
 if (!config.isEmptySegmentAllowed() && rowCount == 0) {
 stepLogger.log("Detect upstream hive table is empty, " + 
"fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/a3fd7d5e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a3fd7d5e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6e377d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6e377d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6e377d8

Branch: refs/heads/yang21-cdh5.7
Commit: f6e377d82f32f2474730060d30351f5de914489c
Parents: a3fd7d5
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Tue Nov 8 14:41:00 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f6e377d8/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6e377d8/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6e377d8/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6e377d8/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a3fd7d5e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a3fd7d5e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a3fd7d5e

Branch: refs/heads/yang21-hbase1.x
Commit: a3fd7d5e442e2a8180eecba65626bea3b7641367
Parents: 1720f1e
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 8 14:38:52 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a3fd7d5e/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[1/3] kylin git commit: KYLIN-2159 Redistribution Hive Table Step always requires row_count filename as 000000_0 [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 471c1e74e -> a3fd7d5e4 (forced update)


KYLIN-2159 Redistribution Hive Table Step always requires row_count filename as 
00_0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1720f1eb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1720f1eb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1720f1eb

Branch: refs/heads/yang21-hbase1.x
Commit: 1720f1eb8aea8bf6807b654558508b261d679bf7
Parents: d9bd93c
Author: shaofengshi 
Authored: Tue Nov 8 12:41:05 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 8 12:42:34 2016 +0800

--
 .../org/apache/kylin/source/hive/HiveMRInput.java  | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1720f1eb/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 95828a0..52f474a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -24,6 +24,7 @@ import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.Job;
@@ -310,10 +311,22 @@ public class HiveMRInput implements IMRInput {
 KylinConfig config = getCubeSpecificConfig();
 
 try {
-
 computeRowCount(config.getCliCommandExecutor());
+Path rowCountFile = null;
+Path rowCountFolder = new Path(getRowCountOutputDir());
+FileSystem fs = FileSystem.get(rowCountFolder.toUri(), 
HadoopUtil.getCurrentConfiguration());
+for (FileStatus stat : fs.listStatus(rowCountFolder)) {
+if (stat.isDirectory() == false && 
stat.getPath().getName().startsWith("")) {
+rowCountFile = stat.getPath();
+logger.debug("Finding file " + rowCountFile);
+break;
+}
+}
+
+if (rowCountFile == null) {
+return new ExecuteResult(ExecuteResult.State.ERROR, "No 
row count file found in '" + getRowCountOutputDir() + "'");
+}
 
-Path rowCountFile = new Path(getRowCountOutputDir(), 
"00_0");
 long rowCount = readRowCountFromFile(rowCountFile);
 if (!config.isEmptySegmentAllowed() && rowCount == 0) {
 stepLogger.log("Detect upstream hive table is empty, " + 
"fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/a3fd7d5e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a3fd7d5e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

kylin git commit: KYLIN-2165 skip redistribute if row count is 0

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master dc1866a8f -> 408f9d43d


KYLIN-2165 skip redistribute if row count is 0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/408f9d43
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/408f9d43
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/408f9d43

Branch: refs/heads/master
Commit: 408f9d43d309eedf19361f0a701213214e436cdd
Parents: dc1866a
Author: shaofengshi 
Authored: Tue Nov 8 14:26:53 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 8 14:26:53 2016 +0800

--
 .../java/org/apache/kylin/source/hive/HiveMRInput.java| 10 +++---
 1 file changed, 7 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/408f9d43/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 67ceffc..9e9dc25 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -282,9 +282,13 @@ public class HiveMRInput implements IMRInput {
 try {
 long rowCount = computeRowCount(database, tableName);
 logger.debug("Row count of table '" + intermediateTable + "' 
is " + rowCount);
-if (!config.isEmptySegmentAllowed() && rowCount == 0) {
-stepLogger.log("Detect upstream hive table is empty, " + 
"fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");
-return new ExecuteResult(ExecuteResult.State.ERROR, 
stepLogger.getBufferedLog());
+if (rowCount == 0) {
+if (!config.isEmptySegmentAllowed()) {
+stepLogger.log("Detect upstream hive table is empty, " 
+ "fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");
+return new ExecuteResult(ExecuteResult.State.ERROR, 
stepLogger.getBufferedLog());
+} else {
+return new ExecuteResult(ExecuteResult.State.SUCCEED, 
"Row count is 0, no need to redistribute");
+}
 }
 
 int mapperInputRows = config.getHadoopJobMapperInputRows();

kylin git commit: KYLIN-2167 FactDistinctColumnsReducer may get wrong max/min partition col value

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2135 e24f054bf -> cd439a96b


KYLIN-2167 FactDistinctColumnsReducer may get wrong max/min partition col value


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd439a96
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd439a96
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd439a96

Branch: refs/heads/KYLIN-2135
Commit: cd439a96b49a8142c0e7cf8fe8ce1a20d392
Parents: e24f054
Author: shaofengshi 
Authored: Tue Nov 8 14:23:11 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 8 14:23:11 2016 +0800

--
 .../mr/steps/FactDistinctColumnsReducer.java|  8 
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  | 40 +---
 2 files changed, 18 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cd439a96/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index a99b857..6703019 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -152,14 +152,6 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
-colValues.set(1, value);
-} else {
-colValues.add(value);
-}
 } else {
 colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, 
key.getLength() - 1)));
 if (colValues.size() == 100) { //spill every 1 million

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd439a96/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 7b9746c..f28e287 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
+import java.text.ParseException;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -82,23 +83,8 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 
 private void updateTimeRange(CubeSegment segment) throws IOException {
 final TblColRef partitionCol = 
segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
-final String factDistinctPath = 
this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
-final ReadableTable readableTable = new DFSFileTable(factDistinctPath 
+ "/" + partitionCol.getName(), -1);
-final ReadableTable.TableReader tableReader = 
readableTable.getReader();
-String minValue = null, maxValue = null;
-try {
-while (tableReader.next()) {
-if (minValue == null) {
-minValue = tableReader.getRow()[0];
-}
-maxValue = tableReader.getRow()[0];
-}
-} finally {
-IOUtils.closeQuietly(tableReader);
-}
-
 final DataType partitionColType = partitionCol.getType();
-FastDateFormat dateFormat;
+final FastDateFormat dateFormat;
 if (partitionColType.isDate()) {
 dateFormat = 
DateFormat.getDateFormat(DateFormat.DEFAULT_DATE_PATTERN);
 } else if (partitionColType.isDatetime() || 
partitionColType.isTimestamp()) {
@@ -113,14 +99,24 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 throw new IllegalStateException("Type " + partitionColType + " is 
not valid partition column type");
 }
 
+final String factDistinctPath = 
this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
+final ReadableTable readableTable = new DFSFileTable(factDistinctPath 
+ "/" + 

kylin git commit: KYLIN-2167 FactDistinctColumnsReducer may get wrong max/min partition col value

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 cbee43ba3 -> 55bc8fb6e


KYLIN-2167 FactDistinctColumnsReducer may get wrong max/min partition col value


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55bc8fb6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55bc8fb6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55bc8fb6

Branch: refs/heads/v1.6.0-rc1
Commit: 55bc8fb6e863760283a335b2b6e969a39f2c9aca
Parents: cbee43b
Author: shaofengshi 
Authored: Tue Nov 8 14:18:49 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 8 14:18:49 2016 +0800

--
 .../mr/steps/FactDistinctColumnsReducer.java|  8 
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  | 47 +---
 2 files changed, 22 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/55bc8fb6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 2889ba8..97d3829 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -130,14 +130,6 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
-colValues.set(1, value);
-} else {
-colValues.add(value);
-}
 } else {
 colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, 
key.getLength() - 1)));
 if (colValues.size() == 100) { //spill every 1 million

http://git-wip-us.apache.org/repos/asf/kylin/blob/55bc8fb6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 4e1be57..d6302b5 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -89,27 +89,8 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 final String outputPath = 
this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
 final Path outputFile = new Path(outputPath, partitionCol.getName());
 
-String minValue = null, maxValue = null, currentValue = null;
-FSDataInputStream inputStream = null;
-BufferedReader bufferedReader = null;
-try {
-FileSystem fs = HadoopUtil.getFileSystem(outputPath);
-inputStream = fs.open(outputFile);
-bufferedReader = new BufferedReader(new 
InputStreamReader(inputStream));
-minValue = currentValue = bufferedReader.readLine();
-while (currentValue != null) {
-maxValue = currentValue;
-currentValue = bufferedReader.readLine();
-}
-} catch (IOException e) {
-throw e;
-} finally {
-IOUtils.closeQuietly(bufferedReader);
-IOUtils.closeQuietly(inputStream);
-}
-
 final DataType partitionColType = partitionCol.getType();
-FastDateFormat dateFormat;
+final FastDateFormat dateFormat;
 if (partitionColType.isDate()) {
 dateFormat = 
DateFormat.getDateFormat(DateFormat.DEFAULT_DATE_PATTERN);
 } else if (partitionColType.isDatetime() || 
partitionColType.isTimestamp()) {
@@ -124,14 +105,30 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 throw new IllegalStateException("Type " + partitionColType + " is 
not valid partition column type");
 }
 
+long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
+String currentValue;
+FSDataInputStream inputStream = null;
+BufferedReader bufferedReader = null;
 try {
-long startTime = dateFormat.parse(minValue).getTime();
-long endTime = 

kylin git commit: KYLIN-2135 update UpdateCubeInfoAfterBuildStep for the new folder structure

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2135 0a20a9b09 -> e24f054bf


KYLIN-2135 update UpdateCubeInfoAfterBuildStep for the new folder structure


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e24f054b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e24f054b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e24f054b

Branch: refs/heads/KYLIN-2135
Commit: e24f054bf1ea130c6a15d19e5ba76728b5a391f3
Parents: 0a20a9b
Author: shaofengshi 
Authored: Tue Nov 8 13:54:35 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 8 13:54:35 2016 +0800

--
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  | 35 +++-
 1 file changed, 12 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e24f054b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 4e1be57..7b9746c 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -18,22 +18,17 @@
 
 package org.apache.kylin.engine.mr.steps;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.FastDateFormat;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.DFSFileTable;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
@@ -41,6 +36,7 @@ import org.apache.kylin.job.execution.ExecutableContext;
 import org.apache.kylin.job.execution.ExecuteResult;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.source.ReadableTable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -86,26 +82,19 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 
 private void updateTimeRange(CubeSegment segment) throws IOException {
 final TblColRef partitionCol = 
segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
-final String outputPath = 
this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
-final Path outputFile = new Path(outputPath, partitionCol.getName());
-
-String minValue = null, maxValue = null, currentValue = null;
-FSDataInputStream inputStream = null;
-BufferedReader bufferedReader = null;
+final String factDistinctPath = 
this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
+final ReadableTable readableTable = new DFSFileTable(factDistinctPath 
+ "/" + partitionCol.getName(), -1);
+final ReadableTable.TableReader tableReader = 
readableTable.getReader();
+String minValue = null, maxValue = null;
 try {
-FileSystem fs = HadoopUtil.getFileSystem(outputPath);
-inputStream = fs.open(outputFile);
-bufferedReader = new BufferedReader(new 
InputStreamReader(inputStream));
-minValue = currentValue = bufferedReader.readLine();
-while (currentValue != null) {
-maxValue = currentValue;
-currentValue = bufferedReader.readLine();
+while (tableReader.next()) {
+if (minValue == null) {
+minValue = tableReader.getRow()[0];
+}
+maxValue = tableReader.getRow()[0];
 }
-} catch (IOException e) {
-throw e;
 } finally {
-IOUtils.closeQuietly(bufferedReader);
-IOUtils.closeQuietly(inputStream);
+IOUtils.closeQuietly(tableReader);
 }
 
 final DataType partitionColType = partitionCol.getType();

kylin git commit: KYLIN-2159 Redistribution Hive Table Step always requires row_count filename as 000000_0

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 da826094b -> cbee43ba3


KYLIN-2159 Redistribution Hive Table Step always requires row_count filename as 
00_0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cbee43ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cbee43ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cbee43ba

Branch: refs/heads/v1.6.0-rc1
Commit: cbee43ba30bd2fe23d1155ef2c4f56197ddedd3c
Parents: da82609
Author: shaofengshi 
Authored: Tue Nov 8 12:41:05 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 8 12:43:23 2016 +0800

--
 .../org/apache/kylin/source/hive/HiveMRInput.java  | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cbee43ba/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index f536cbb..c2a4816 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -26,6 +26,7 @@ import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.Job;
@@ -330,10 +331,22 @@ public class HiveMRInput implements IMRInput {
 KylinConfig config = getCubeSpecificConfig();
 
 try {
-
 computeRowCount(config.getCliCommandExecutor());
+Path rowCountFile = null;
+Path rowCountFolder = new Path(getRowCountOutputDir());
+FileSystem fs = FileSystem.get(rowCountFolder.toUri(), 
HadoopUtil.getCurrentConfiguration());
+for (FileStatus stat : fs.listStatus(rowCountFolder)) {
+if (stat.isDirectory() == false && 
stat.getPath().getName().startsWith("")) {
+rowCountFile = stat.getPath();
+logger.debug("Finding file " + rowCountFile);
+break;
+}
+}
+
+if (rowCountFile == null) {
+return new ExecuteResult(ExecuteResult.State.ERROR, "No 
row count file found in '" + getRowCountOutputDir() + "'");
+}
 
-Path rowCountFile = new Path(getRowCountOutputDir(), 
"00_0");
 long rowCount = readRowCountFromFile(rowCountFile);
 if (!config.isEmptySegmentAllowed() && rowCount == 0) {
 stepLogger.log("Detect upstream hive table is empty, " + 
"fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");

kylin git commit: KYLIN-2159 Redistribution Hive Table Step always requires row_count filename as 000000_0

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/yang21 d9bd93c48 -> 1720f1eb8


KYLIN-2159 Redistribution Hive Table Step always requires row_count filename as 
00_0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1720f1eb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1720f1eb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1720f1eb

Branch: refs/heads/yang21
Commit: 1720f1eb8aea8bf6807b654558508b261d679bf7
Parents: d9bd93c
Author: shaofengshi 
Authored: Tue Nov 8 12:41:05 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 8 12:42:34 2016 +0800

--
 .../org/apache/kylin/source/hive/HiveMRInput.java  | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1720f1eb/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 95828a0..52f474a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -24,6 +24,7 @@ import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.Job;
@@ -310,10 +311,22 @@ public class HiveMRInput implements IMRInput {
 KylinConfig config = getCubeSpecificConfig();
 
 try {
-
 computeRowCount(config.getCliCommandExecutor());
+Path rowCountFile = null;
+Path rowCountFolder = new Path(getRowCountOutputDir());
+FileSystem fs = FileSystem.get(rowCountFolder.toUri(), 
HadoopUtil.getCurrentConfiguration());
+for (FileStatus stat : fs.listStatus(rowCountFolder)) {
+if (stat.isDirectory() == false && 
stat.getPath().getName().startsWith("")) {
+rowCountFile = stat.getPath();
+logger.debug("Finding file " + rowCountFile);
+break;
+}
+}
+
+if (rowCountFile == null) {
+return new ExecuteResult(ExecuteResult.State.ERROR, "No 
row count file found in '" + getRowCountOutputDir() + "'");
+}
 
-Path rowCountFile = new Path(getRowCountOutputDir(), 
"00_0");
 long rowCount = readRowCountFromFile(rowCountFile);
 if (!config.isEmptySegmentAllowed() && rowCount == 0) {
 stepLogger.log("Detect upstream hive table is empty, " + 
"fail the job because \"kylin.job.allow.empty.segment\" = \"false\"");

kylin git commit: KYLIN-2166 Unclosed HBaseAdmin in StorageCleanupJob#cleanUnusedHBaseTables

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master 7211d9215 -> dc1866a8f


KYLIN-2166 Unclosed HBaseAdmin in StorageCleanupJob#cleanUnusedHBaseTables


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc1866a8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc1866a8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc1866a8

Branch: refs/heads/master
Commit: dc1866a8f21956acea6d489e37a731e87faf4502
Parents: 7211d92
Author: lidongsjtu 
Authored: Mon Nov 7 23:59:15 2016 +0800
Committer: lidongsjtu 
Committed: Mon Nov 7 23:59:55 2016 +0800

--
 .../apache/kylin/tool/StorageCleanupJob.java| 81 ++--
 1 file changed, 40 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dc1866a8/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java 
b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 3f82e94..4252e74 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -82,55 +82,54 @@ public class StorageCleanupJob extends AbstractApplication {
 private void cleanUnusedHBaseTables(Configuration conf) throws IOException 
{
 CubeManager cubeMgr = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
 // get all kylin hbase tables
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
-String tableNamePrefix = 
IRealizationConstants.SharedHbaseStorageLocationPrefix;
-HTableDescriptor[] tableDescriptors = 
hbaseAdmin.listTables(tableNamePrefix + ".*");
-List allTablesNeedToBeDropped = new ArrayList();
-for (HTableDescriptor desc : tableDescriptors) {
-String host = desc.getValue(IRealizationConstants.HTableTag);
-if 
(KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host))
 {
-//only take care htables that belongs to self, and created 
more than 2 days
-
allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
+try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) {
+String tableNamePrefix = 
IRealizationConstants.SharedHbaseStorageLocationPrefix;
+HTableDescriptor[] tableDescriptors = 
hbaseAdmin.listTables(tableNamePrefix + ".*");
+List allTablesNeedToBeDropped = new ArrayList();
+for (HTableDescriptor desc : tableDescriptors) {
+String host = desc.getValue(IRealizationConstants.HTableTag);
+if 
(KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host))
 {
+//only take care htables that belongs to self, and created 
more than 2 days
+
allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
+}
 }
-}
 
-// remove every segment htable from drop list
-for (CubeInstance cube : cubeMgr.listAllCubes()) {
-for (CubeSegment seg : cube.getSegments()) {
-String tablename = seg.getStorageLocationIdentifier();
-if (allTablesNeedToBeDropped.contains(tablename)) {
-allTablesNeedToBeDropped.remove(tablename);
-logger.info("Exclude table " + tablename + " from drop 
list, as the table belongs to cube " + cube.getName() + " with status " + 
cube.getStatus());
+// remove every segment htable from drop list
+for (CubeInstance cube : cubeMgr.listAllCubes()) {
+for (CubeSegment seg : cube.getSegments()) {
+String tablename = seg.getStorageLocationIdentifier();
+if (allTablesNeedToBeDropped.contains(tablename)) {
+allTablesNeedToBeDropped.remove(tablename);
+logger.info("Exclude table " + tablename + " from drop 
list, as the table belongs to cube " + cube.getName() + " with status " + 
cube.getStatus());
+}
 }
 }
-}
 
-if (delete == true) {
-// drop tables
-ExecutorService executorService = 
Executors.newSingleThreadExecutor();
-for (String htableName : allTablesNeedToBeDropped) {
-FutureTask futureTask = new FutureTask(new 
DeleteHTableRunnable(hbaseAdmin, htableName));
-executorService.execute(futureTask);
-try {
-futureTask.get(deleteTimeout, TimeUnit.MINUTES);
-} catch (TimeoutException e) {
-

[2/2] kylin git commit: KYLIN-2006 minor revision

[liyang]

KYLIN-2006 minor revision


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db991056
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db991056
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db991056

Branch: refs/heads/KYLIN-2006
Commit: db991056848b15d75f51f82ee83ccab52ecc266a
Parents: 70d75fa
Author: Yang Li 
Authored: Mon Nov 7 20:59:25 2016 +0800
Committer: Yang Li 
Committed: Mon Nov 7 20:59:25 2016 +0800

--
 .../impl/threadpool/DistributedScheduler.java   |  3 +--
 .../kylin/job/lock/DistributedJobLock.java  |  7 +-
 .../org/apache/kylin/job/lock/DoWatchLock.java  | 23 
 .../hbase/util/ZookeeperDistributedJobLock.java |  1 -
 4 files changed, 7 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db991056/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index 11709c7..17df119 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -46,7 +46,6 @@ import org.apache.kylin.job.execution.Executable;
 import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.execution.Output;
 import org.apache.kylin.job.lock.DistributedJobLock;
-import org.apache.kylin.job.lock.DoWatchLock;
 import org.apache.kylin.job.lock.JobLock;
 import org.apache.kylin.job.manager.ExecutableManager;
 import org.slf4j.Logger;
@@ -209,7 +208,7 @@ public class DistributedScheduler implements 
Scheduler, Conn
 }
 
 //when the segment lock released but the segment related job still 
running, resume the job.
-private class DoWatchImpl implements DoWatchLock {
+private class DoWatchImpl implements 
org.apache.kylin.job.lock.DistributedJobLock.DoWatchLock {
 private String serverName;
 
 public DoWatchImpl(String serverName) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/db991056/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java 
b/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java
index 5ba8426..9335e56 100644
--- a/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java
+++ b/core-job/src/main/java/org/apache/kylin/job/lock/DistributedJobLock.java
@@ -21,9 +21,14 @@ package org.apache.kylin.job.lock;
 import java.util.concurrent.ExecutorService;
 
 public interface DistributedJobLock extends JobLock {
-boolean lockWithName(String cubeName, String serverName);
+
+boolean lockWithName(String name, String serverName);
 
 void unlockWithName(String name);
 
 void watchLock(ExecutorService pool, DoWatchLock doWatch);
+
+public interface DoWatchLock {
+void doWatch(String path, String data);
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/db991056/core-job/src/main/java/org/apache/kylin/job/lock/DoWatchLock.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/lock/DoWatchLock.java 
b/core-job/src/main/java/org/apache/kylin/job/lock/DoWatchLock.java
deleted file mode 100644
index 08c13f9..000
--- a/core-job/src/main/java/org/apache/kylin/job/lock/DoWatchLock.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.lock;
-
-public interface DoWatchLock {
-void doWatch(String path, String data);
-}

[1/2] kylin git commit: KYLIN-2006 Make job engine distributed and HA

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2006 [created] db9910568


KYLIN-2006 Make job engine distributed and HA

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/70d75fac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/70d75fac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/70d75fac

Branch: refs/heads/KYLIN-2006
Commit: 70d75facaee50a6cf6ad77f722b2de680e87e5e1
Parents: 7211d92
Author: kangkaisen 
Authored: Mon Sep 5 20:15:23 2016 +0800
Committer: Yang Li 
Committed: Mon Nov 7 20:26:09 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   1 +
 .../impl/threadpool/DistributedScheduler.java   | 349 +++
 .../kylin/job/lock/DistributedJobLock.java  |  29 ++
 .../org/apache/kylin/job/lock/DoWatchLock.java  |  23 ++
 .../kylin/job/manager/ExecutableManager.java|  23 ++
 .../kylin/job/BaseTestDistributedScheduler.java | 226 
 .../apache/kylin/job/ContextTestExecutable.java |  51 +++
 .../job/ITDistributedSchedulerBaseTest.java |  90 +
 .../job/ITDistributedSchedulerTakeOverTest.java |  60 
 .../kylin/rest/controller/JobController.java|  62 +---
 .../apache/kylin/rest/service/CubeService.java  |   4 +
 .../apache/kylin/rest/service/JobService.java   |  96 -
 .../hbase/util/ZookeeperDistributedJobLock.java | 230 
 13 files changed, 1182 insertions(+), 62 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/70d75fac/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d9d10bb..71d3a38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -750,6 +750,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public Map getSchedulers() {
 Map r = 
convertKeyToInteger(getPropertiesByPrefix("kylin.scheduler."));
 r.put(0, "org.apache.kylin.job.impl.threadpool.DefaultScheduler");
+r.put(2, "org.apache.kylin.job.impl.threadpool.DistributedScheduler");
 return r;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/70d75fac/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
new file mode 100644
index 000..11709c7
--- /dev/null
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.impl.threadpool;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.state.ConnectionState;
+import org.apache.curator.framework.state.ConnectionStateListener;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.job.Scheduler;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import 

kylin git commit: KYLIN-2163 use -v opt instead of env variable for kylin.sh

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 0efc45da0 -> 7211d9215


KYLIN-2163 use -v opt instead of env variable for kylin.sh

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7211d921
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7211d921
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7211d921

Branch: refs/heads/master
Commit: 7211d92158c72979719df68659863ff985eafaa4
Parents: 0efc45d
Author: etherge 
Authored: Sat Nov 5 21:32:55 2016 +0800
Committer: Yang Li 
Committed: Mon Nov 7 20:13:47 2016 +0800

--
 build/bin/header.sh | 14 ++
 build/bin/kylin.sh  |  8 +---
 2 files changed, 19 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7211d921/build/bin/header.sh
--
diff --git a/build/bin/header.sh b/build/bin/header.sh
index 949ea95..8a6fc00 100644
--- a/build/bin/header.sh
+++ b/build/bin/header.sh
@@ -19,6 +19,20 @@
 
 # source me
 
+verbose=${verbose:-""}
+
+while getopts ":v" opt; do
+case $opt in
+v)
+echo "Turn on verbose mode." >&2
+verbose=true
+;;
+\?)
+echo "Invalid option: -$OPTARG" >&2
+;;
+esac
+done
+
 if [[ "$dir" == "" ]]
 then
dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)

http://git-wip-us.apache.org/repos/asf/kylin/blob/7211d921/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index f45c1dd..dedb552 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -18,9 +18,11 @@
 #
 
 # set verbose=true to print more logs during start up
-verbose=${verbose:-""}
 
-source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh $@
+if [ "$verbose" = true ]; then
+shift
+fi
 
 source ${dir}/check-env.sh
 mkdir -p ${KYLIN_HOME}/logs
@@ -162,5 +164,5 @@ then
 exec hbase ${KYLIN_EXTRA_START_OPTS} 
-Dkylin.hive.dependency=${hive_dependency} 
-Dkylin.hbase.dependency=${hbase_dependency} 
-Dlog4j.configuration=kylin-log4j.properties "$@"
 
 else
-quit "Usage: 'kylin.sh start' or 'kylin.sh stop'"
+quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop'"
 fi

kylin git commit: KYLIN 2119 Wrong chart value and sort when process scientific notation

[zhongjian]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 f2003 -> da826094b


KYLIN 2119 Wrong chart value and sort when process scientific notation

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da826094
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da826094
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da826094

Branch: refs/heads/v1.6.0-rc1
Commit: da826094bf86759aee0ceb82eae7fc1aed88edf6
Parents: f2003ee
Author: chenzhx <346839...@qq.com>
Authored: Mon Nov 7 18:40:26 2016 +0800
Committer: Jason 
Committed: Mon Nov 7 18:53:06 2016 +0800

--
 webapp/app/index.html  |  2 +-
 webapp/app/js/controllers/query.js | 11 +--
 webapp/app/js/services/graph.js| 10 +-
 webapp/app/js/utils/utils.js   | 20 
 4 files changed, 39 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/da826094/webapp/app/index.html
--
diff --git a/webapp/app/index.html b/webapp/app/index.html
index 8fb2cb9..ad881d5 100644
--- a/webapp/app/index.html
+++ b/webapp/app/index.html
@@ -163,7 +163,7 @@
 
 
 
-
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/da826094/webapp/app/js/controllers/query.js
--
diff --git a/webapp/app/js/controllers/query.js 
b/webapp/app/js/controllers/query.js
index dede294..6be915b 100644
--- a/webapp/app/js/controllers/query.js
+++ b/webapp/app/js/controllers/query.js
@@ -19,7 +19,7 @@
 'use strict';
 
 KylinApp
-.controller('QueryCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, $modal, MessageService, 
$domUtilityService, $timeout, TableService,SweetAlert) {
+.controller('QueryCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, $modal, MessageService, 
$domUtilityService, $timeout, TableService, SweetAlert, VdmUtil) {
 $scope.mainPanel = 'query';
 $scope.rowsPerPage = 5;
 $scope.base64 = $base64;
@@ -207,7 +207,14 @@ KylinApp
 } else {
 oneQuery.result.data = data;
 }
-
+angular.forEach(oneQuery.result.data,function(row,index){
+angular.forEach(row,function(column,value){
+var float =VdmUtil.SCToFloat(column);
+if (float!=""){
+
oneQuery.result.data[index][value]=parseFloat(float);
+}
+});
+});
 $scope.curQuery.result.isResponsePartial = result.partial;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/da826094/webapp/app/js/services/graph.js
--
diff --git a/webapp/app/js/services/graph.js b/webapp/app/js/services/graph.js
index b191c50..dc69519 100644
--- a/webapp/app/js/services/graph.js
+++ b/webapp/app/js/services/graph.js
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-KylinApp.service('GraphService', function (GraphBuilder) {
+KylinApp.service('GraphService', function (GraphBuilder, VdmUtil) {
 
   this.buildGraph = function (query) {
 var graphData = null;
@@ -27,6 +27,14 @@ KylinApp.service('GraphService', function (GraphBuilder) {
   metricsList = metricsList.concat(query.graph.state.metrics);
   angular.forEach(metricsList, function (metrics, index) {
 var aggregatedData = {};
+angular.forEach(query.result.results,function(row,index){
+  angular.forEach(row,function(column,value){
+var float = VdmUtil.SCToFloat(column);
+  if (float!=""){
+query.result.results[index][value]=float;
+  }
+  });
+});
 angular.forEach(query.result.results, function (data, index) {
   aggregatedData[data[dimension.index]] = 
(!!aggregatedData[data[dimension.index]] ? 
aggregatedData[data[dimension.index]] : 0)
   + parseFloat(data[metrics.index].replace(/[^\d\.\-]/g, ""));

http://git-wip-us.apache.org/repos/asf/kylin/blob/da826094/webapp/app/js/utils/utils.js
--
diff --git a/webapp/app/js/utils/utils.js b/webapp/app/js/utils/utils.js
index d838e9e..d25c4e1 100644
--- a/webapp/app/js/utils/utils.js
+++ b/webapp/app/js/utils/utils.js
@@ -63,6 +63,26 @@ KylinApp.factory('VdmUtil', function ($modal, $timeout, 
$location, $anchorScroll
 if (new RegExp("(" + k + ")").test(fmt)) fmt = fmt.replace(RegExp.$1, 
(RegExp.$1.length == 1) ? (o[k]) : 

kylin git commit: KYLIN 2119 Wrong chart value and sort when process scientific notation

[zhongjian]

Repository: kylin
Updated Branches:
  refs/heads/master 76a53da74 -> 0efc45da0


KYLIN 2119 Wrong chart value and sort when process scientific notation

Signed-off-by: Jason 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0efc45da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0efc45da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0efc45da

Branch: refs/heads/master
Commit: 0efc45da0f87d0fd22963a296c40d6806bc48bba
Parents: 76a53da
Author: chenzhx <346839...@qq.com>
Authored: Mon Nov 7 18:40:26 2016 +0800
Committer: Jason 
Committed: Mon Nov 7 18:52:17 2016 +0800

--
 webapp/app/index.html  |  2 +-
 webapp/app/js/controllers/query.js | 11 +--
 webapp/app/js/services/graph.js| 10 +-
 webapp/app/js/utils/utils.js   | 20 
 4 files changed, 39 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0efc45da/webapp/app/index.html
--
diff --git a/webapp/app/index.html b/webapp/app/index.html
index 8fb2cb9..ad881d5 100644
--- a/webapp/app/index.html
+++ b/webapp/app/index.html
@@ -163,7 +163,7 @@
 
 
 
-
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0efc45da/webapp/app/js/controllers/query.js
--
diff --git a/webapp/app/js/controllers/query.js 
b/webapp/app/js/controllers/query.js
index dede294..6be915b 100644
--- a/webapp/app/js/controllers/query.js
+++ b/webapp/app/js/controllers/query.js
@@ -19,7 +19,7 @@
 'use strict';
 
 KylinApp
-.controller('QueryCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, $modal, MessageService, 
$domUtilityService, $timeout, TableService,SweetAlert) {
+.controller('QueryCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, $modal, MessageService, 
$domUtilityService, $timeout, TableService, SweetAlert, VdmUtil) {
 $scope.mainPanel = 'query';
 $scope.rowsPerPage = 5;
 $scope.base64 = $base64;
@@ -207,7 +207,14 @@ KylinApp
 } else {
 oneQuery.result.data = data;
 }
-
+angular.forEach(oneQuery.result.data,function(row,index){
+angular.forEach(row,function(column,value){
+var float =VdmUtil.SCToFloat(column);
+if (float!=""){
+
oneQuery.result.data[index][value]=parseFloat(float);
+}
+});
+});
 $scope.curQuery.result.isResponsePartial = result.partial;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0efc45da/webapp/app/js/services/graph.js
--
diff --git a/webapp/app/js/services/graph.js b/webapp/app/js/services/graph.js
index b191c50..dc69519 100644
--- a/webapp/app/js/services/graph.js
+++ b/webapp/app/js/services/graph.js
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-KylinApp.service('GraphService', function (GraphBuilder) {
+KylinApp.service('GraphService', function (GraphBuilder, VdmUtil) {
 
   this.buildGraph = function (query) {
 var graphData = null;
@@ -27,6 +27,14 @@ KylinApp.service('GraphService', function (GraphBuilder) {
   metricsList = metricsList.concat(query.graph.state.metrics);
   angular.forEach(metricsList, function (metrics, index) {
 var aggregatedData = {};
+angular.forEach(query.result.results,function(row,index){
+  angular.forEach(row,function(column,value){
+var float = VdmUtil.SCToFloat(column);
+  if (float!=""){
+query.result.results[index][value]=float;
+  }
+  });
+});
 angular.forEach(query.result.results, function (data, index) {
   aggregatedData[data[dimension.index]] = 
(!!aggregatedData[data[dimension.index]] ? 
aggregatedData[data[dimension.index]] : 0)
   + parseFloat(data[metrics.index].replace(/[^\d\.\-]/g, ""));

http://git-wip-us.apache.org/repos/asf/kylin/blob/0efc45da/webapp/app/js/utils/utils.js
--
diff --git a/webapp/app/js/utils/utils.js b/webapp/app/js/utils/utils.js
index d838e9e..d25c4e1 100644
--- a/webapp/app/js/utils/utils.js
+++ b/webapp/app/js/utils/utils.js
@@ -63,6 +63,26 @@ KylinApp.factory('VdmUtil', function ($modal, $timeout, 
$location, $anchorScroll
 if (new RegExp("(" + k + ")").test(fmt)) fmt = fmt.replace(RegExp.$1, 
(RegExp.$1.length == 1) ? (o[k]) : (("00" + 

kylin git commit: KYLIN-2165 cleanup old codes

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master b8acf14f5 -> 76a53da74


KYLIN-2165 cleanup old codes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76a53da7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76a53da7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76a53da7

Branch: refs/heads/master
Commit: 76a53da74536f4b3414bd7f5ef3950b9b489b3e8
Parents: b8acf14
Author: shaofengshi 
Authored: Mon Nov 7 18:20:28 2016 +0800
Committer: shaofengshi 
Committed: Mon Nov 7 18:21:08 2016 +0800

--
 .../org/apache/kylin/job/JoinedFlatTable.java   | 10 +--
 .../apache/kylin/job/JoinedFlatTableTest.java   |  2 +-
 .../kylin/rest/controller/CubeController.java   |  2 +-
 .../source/hive/CreateFlatHiveTableStep.java| 72 +---
 .../apache/kylin/source/hive/HiveMRInput.java   | 41 ++-
 .../kylin/source/hive/HiveCmdBuilderTest.java   |  6 +-
 6 files changed, 18 insertions(+), 115 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/76a53da7/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 6c08bc9..b26f50d 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -98,14 +98,14 @@ public class JoinedFlatTable {
 return ddl.toString();
 }
 
-public static String generateInsertDataStatement(IJoinedFlatTableDesc 
flatDesc, JobEngineConfig engineConfig, boolean redistribute) {
+public static String generateInsertDataStatement(IJoinedFlatTableDesc 
flatDesc, JobEngineConfig engineConfig) {
 StringBuilder sql = new StringBuilder();
 sql.append(generateHiveSetStatements(engineConfig));
-sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + 
generateSelectDataStatement(flatDesc, redistribute) + ";").append("\n");
+sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + 
generateSelectDataStatement(flatDesc) + ";").append("\n");
 return sql.toString();
 }
 
-public static String generateSelectDataStatement(IJoinedFlatTableDesc 
flatDesc, boolean redistribute) {
+public static String generateSelectDataStatement(IJoinedFlatTableDesc 
flatDesc) {
 StringBuilder sql = new StringBuilder();
 sql.append("SELECT" + "\n");
 for (int i = 0; i < flatDesc.getAllColumns().size(); i++) {
@@ -117,10 +117,6 @@ public class JoinedFlatTable {
 }
 appendJoinStatement(flatDesc, sql);
 appendWhereStatement(flatDesc, sql);
-if (redistribute == true) {
-TblColRef distCol = flatDesc.getDistributedBy();
-appendDistributeStatement(sql, distCol);
-}
 return sql.toString();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/76a53da7/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java 
b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
index 1fe47f8..0faf22a 100644
--- a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
@@ -77,7 +77,7 @@ public class JoinedFlatTableTest extends 
LocalFileMetadataTestCase {
 
 @Test
 public void testGenerateInsertSql() throws IOException {
-String sqls = 
JoinedFlatTable.generateInsertDataStatement(flatTableDesc, new 
JobEngineConfig(KylinConfig.getInstanceFromEnv()), true);
+String sqls = 
JoinedFlatTable.generateInsertDataStatement(flatTableDesc, new 
JobEngineConfig(KylinConfig.getInstanceFromEnv()));
 System.out.println(sqls);
 
 int length = sqls.length();

http://git-wip-us.apache.org/repos/asf/kylin/blob/76a53da7/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 79739c2..5ca7cb5 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -145,7 +145,7 @@ public class CubeController extends BasicController {
 public GeneralResponse getSql(@PathVariable 

[2/3] kylin git commit: KYLIN-2156: support filters like a != 0

[mahongbin]

KYLIN-2156: support filters like a != 0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a0c4bfba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a0c4bfba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a0c4bfba

Branch: refs/heads/master
Commit: a0c4bfba39859bca5049326f227f4e6137ab106c
Parents: 04ac1c4
Author: Hongbin Ma 
Authored: Thu Nov 3 16:49:42 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 7 14:51:49 2016 +0800

--
 .../main/java/org/apache/kylin/rest/util/QueryUtil.java   | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a0c4bfba/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index eb73e8f..59a4a78 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -95,6 +95,7 @@ public class QueryUtil {
 private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + 
"GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = 
Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + 
"1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!="+ 
S0, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_INTERVAL = Pattern.compile("interval" 
+ SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_CONCAT = 
Pattern.compile("concat\\(.+?\\)");//non-greedy
 private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);
@@ -129,6 +130,15 @@ public class QueryUtil {
 sql = sql.substring(0, m.start()) + " COUNT(1) " + 
sql.substring(m.end());
 }
 
+// Case: !=
+// Replace it with <>
+while (true) {
+m = PTN_NOT_EQ.matcher(sql);
+if (!m.find())
+break;
+sql = sql.substring(0, m.start()) + " <> " + 
sql.substring(m.end());
+}
+
 // ( date '2001-09-28' + interval floor(1) day ) generated by 
cognos
 // calcite only recognizes date '2001-09-28' + interval '1' day
 while (true) {

[3/3] kylin git commit: if outer query does not have aggr, join columns with subquery should not be added to groupbycolumns (squashed)

[mahongbin]

if outer query does not have aggr, join columns with subquery should not be 
added to groupbycolumns (squashed)

fix CI

fix CI

bug fix: if outer query does not have aggr, join columns with subquery should 
not be added to groupbycolumns

bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8acf14f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8acf14f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8acf14f

Branch: refs/heads/master
Commit: b8acf14f5e963e48024cd479b64dcf9f1be4d2e0
Parents: a0c4bfb
Author: Hongbin Ma 
Authored: Thu Nov 3 18:32:19 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 7 15:01:15 2016 +0800

--
 .../main/java/org/apache/kylin/cube/cuboid/Cuboid.java |  5 +
 .../apache/kylin/metadata/realization/SQLDigest.java   | 13 -
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java |  3 +++
 .../org/apache/kylin/storage/hbase/ITStorageTest.java  |  4 +++-
 .../org/apache/kylin/query/relnode/OLAPContext.java|  7 +++
 .../org/apache/kylin/query/relnode/OLAPJoinRel.java|  5 ++---
 .../kylin/storage/hbase/cube/v1/CubeStorageQuery.java  |  4 
 7 files changed, 32 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 855aad6..7503fbf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -89,6 +89,11 @@ public class Cuboid implements Comparable {
 }
 
 public static boolean isValid(CubeDesc cube, long cuboidID) {
+//base cuboid is always valid
+if (cuboidID == getBaseCuboidId(cube)) {
+return true;
+}
+
 for (AggregationGroup agg : cube.getAggregationGroups()) {
 if (isValid(agg, cuboidID)) {
 return true;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 90fa53a..83fc05c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -51,6 +51,7 @@ public class SQLDigest {
 public List joinDescs;
 public Set allColumns;
 public List groupbyColumns;
+public Set subqueryJoinParticipants;
 public Set filterColumns;
 public Set metricColumns;
 public List aggregations; // storage level measure type, on 
top of which various sql aggr function may apply
@@ -60,13 +61,14 @@ public class SQLDigest {
 public boolean isRawQuery;
 
 public SQLDigest(String factTable, TupleFilter filter, List 
joinDescs, Set allColumns, //
-List groupbyColumns, Set filterColumns, 
Set metricColumns, //
+List groupbyColumns, Set 
subqueryJoinParticipants, Set filterColumns, Set 
metricColumns, //
 List aggregations, List aggrSqlCalls, 
List sortColumns, List sortOrders) {
 this.factTable = factTable;
 this.filter = filter;
 this.joinDescs = joinDescs;
 this.allColumns = allColumns;
 this.groupbyColumns = groupbyColumns;
+this.subqueryJoinParticipants = subqueryJoinParticipants;
 this.filterColumns = filterColumns;
 this.metricColumns = metricColumns;
 this.aggregations = aggregations;
@@ -82,6 +84,15 @@ public class SQLDigest {
 //the reason to choose aggregations rather than metricColumns is 
because the former is set earlier at implOLAP
 }
 
+public void includeSubqueryJoinParticipants() {
+if (this.isRawQuery) {
+this.allColumns.addAll(this.subqueryJoinParticipants);
+} else {
+this.groupbyColumns.addAll(this.subqueryJoinParticipants);
+this.allColumns.addAll(this.subqueryJoinParticipants);
+}
+}
+
 @Override
 public String toString() {
 return "fact table " + this.factTable + "," + //

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 

[1/3] kylin git commit: KYLIN-1945 Cuboid.translateToValidCuboid method throw exception while cube building or query execute

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master a9fe953df -> b8acf14f5


KYLIN-1945 Cuboid.translateToValidCuboid method throw exception while cube 
building or query execute


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/04ac1c49
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/04ac1c49
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/04ac1c49

Branch: refs/heads/master
Commit: 04ac1c492ecb4dea02c3f952e3cce7bfca2b4b8d
Parents: a9fe953
Author: Hongbin Ma 
Authored: Thu Nov 3 16:01:04 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 7 14:51:39 2016 +0800

--
 .../main/java/org/apache/kylin/cube/cuboid/Cuboid.java| 10 +-
 .../org/apache/kylin/cube/model/AggregationGroup.java |  4 
 2 files changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/04ac1c49/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index e4e718d..855aad6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -89,16 +89,11 @@ public class Cuboid implements Comparable {
 }
 
 public static boolean isValid(CubeDesc cube, long cuboidID) {
-if (cuboidID == getBaseCuboidId(cube)) {
-return true;
-}
-
 for (AggregationGroup agg : cube.getAggregationGroups()) {
 if (isValid(agg, cuboidID)) {
 return true;
 }
 }
-
 return false;
 }
 
@@ -231,6 +226,11 @@ public class Cuboid implements Comparable {
 if ((cuboidID & mandatoryColumnMask) != mandatoryColumnMask) {
 return false;
 } else {
+//base cuboid is always valid
+if (cuboidID == getBaseCuboidId(agg.getCubeDesc())) {
+return true;
+}
+
 //cuboid with only mandatory columns maybe valid
 return agg.isMandatoryOnlyValid() || (cuboidID & 
~mandatoryColumnMask) != 0;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/04ac1c49/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 6e76ac3..027d185 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -293,4 +293,8 @@ public class AggregationGroup {
 public boolean isMandatoryOnlyValid() {
 return isMandatoryOnlyValid;
 }
+
+public CubeDesc getCubeDesc() {
+return cubeDesc;
+}
 }

[10/10] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4df0043
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4df0043
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4df0043

Branch: refs/heads/yang21-cdh5.7
Commit: d4df0043e05ffaebc34e1e1e1f9e726bef7d673a
Parents: 471c1e7
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Li Yang 
Committed: Mon Nov 7 15:53:40 2016 +

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d4df0043/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/d4df0043/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/d4df0043/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d4df0043/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[09/10] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/471c1e74
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/471c1e74
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/471c1e74

Branch: refs/heads/yang21-cdh5.7
Commit: 471c1e74e4fdb25f55a58efb184ed67267d8bf0a
Parents: d9bd93c
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Mon Nov 7 15:51:24 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/471c1e74/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[04/10] kylin git commit: KYLIN-2163 refine kylin scripts

[liyang]

KYLIN-2163 refine kylin scripts


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8e31ec5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8e31ec5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8e31ec5

Branch: refs/heads/yang21-cdh5.7
Commit: f8e31ec54dab0a77b3ba2885cf229a99f1c8560c
Parents: b4f9179
Author: Yang Li 
Authored: Sat Nov 5 12:00:53 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 5 12:00:53 2016 +0800

--
 build/bin/check-env.sh | 17 --
 build/bin/diag.sh  |  7 +---
 build/bin/find-hbase-dependency.sh |  7 ++--
 build/bin/find-hive-dependency.sh  | 31 +++---
 build/bin/find-kafka-dependency.sh |  2 ++
 build/bin/get-properties.sh|  2 ++
 build/bin/header.sh| 41 
 build/bin/kylin.sh | 57 +
 build/bin/metastore.sh |  7 +---
 build/bin/sample.sh|  3 +-
 build/bin/setenv.sh| 12 ---
 11 files changed, 99 insertions(+), 87 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/check-env.sh
--
diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh
index 1597c53..9cb7636 100644
--- a/build/bin/check-env.sh
+++ b/build/bin/check-env.sh
@@ -17,38 +17,33 @@
 # limitations under the License.
 #
 
-#by source
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
 if [ -z "$KYLIN_HOME" ]
 then
-echo 'please make sure KYLIN_HOME has been set'
-exit 1
+quit 'Please make sure KYLIN_HOME has been set'
 else
 echo "KYLIN_HOME is set to ${KYLIN_HOME}"
 fi
 
 if [ -z "$(command -v hbase version)" ]
 then
-echo "Please make sure the user has the privilege to run hbase shell"
-exit 1
+quit "Please make sure the user has the privilege to run hbase shell"
 fi
 
 if [ -z "$(command -v hive --version)" ]
 then
-echo "Please make sure the user has the privilege to run hive shell"
-exit 1
+quit "Please make sure the user has the privilege to run hive shell"
 fi
 
 if [ -z "$(command -v hadoop version)" ]
 then
-echo "Please make sure the user has the privilege to run hadoop shell"
-exit 1
+quit "Please make sure the user has the privilege to run hadoop shell"
 fi
 
 WORKING_DIR=`sh $KYLIN_HOME/bin/get-properties.sh kylin.hdfs.working.dir`
 hadoop fs -mkdir -p $WORKING_DIR
 if [ $? != 0 ]
 then
-echo "failed to create $WORKING_DIR, Please make sure the user has right 
to access $WORKING_DIR"
-exit 1
+quit "Failed to create $WORKING_DIR. Please make sure the user has right 
to access $WORKING_DIR"
 fi

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/diag.sh
--
diff --git a/build/bin/diag.sh b/build/bin/diag.sh
index 91ba92b..5f3cec3 100644
--- a/build/bin/diag.sh
+++ b/build/bin/diag.sh
@@ -17,13 +17,8 @@
 # limitations under the License.
 #
 
-# We should set KYLIN_HOME here for multiple tomcat instances that are on the 
same node.
-# In addition, we should set a KYLIN_HOME for the global use as normal.
-KYLIN_HOME=`dirname $0`/..
-export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
-dir="$KYLIN_HOME/bin"
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-source ${dir}/check-env.sh
 mkdir -p ${KYLIN_HOME}/logs
 
 tomcat_root=${dir}/../tomcat

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/find-hbase-dependency.sh
--
diff --git a/build/bin/find-hbase-dependency.sh 
b/build/bin/find-hbase-dependency.sh
index ea9a2e2..7dbb53b 100644
--- a/build/bin/find-hbase-dependency.sh
+++ b/build/bin/find-hbase-dependency.sh
@@ -17,6 +17,8 @@
 # limitations under the License.
 #
 
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
 hbase_classpath=`hbase classpath`
 
 # special handling for Amazon EMR, to prevent re-init of hbase-setenv
@@ -38,10 +40,9 @@ done
 
 if [ -z "$hbase_common_path" ]
 then
-echo "hbase-common lib not found"
-exit 1
+quit "hbase-common lib not found"
 fi
 
 hbase_dependency=${hbase_common_path}
-echo "hbase dependency: $hbase_dependency"
+verbose "hbase dependency: $hbase_dependency"
 export hbase_dependency

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/find-hive-dependency.sh
--
diff --git a/build/bin/find-hive-dependency.sh 
b/build/bin/find-hive-dependency.sh
index d17b47e..69b63a4 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -17,24 +17,18 @@
 # 

[05/10] kylin git commit: KYLIN-2162 log errors when cube desc broken

[liyang]

KYLIN-2162 log errors when cube desc broken

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b2efcd7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b2efcd7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b2efcd7

Branch: refs/heads/yang21-cdh5.7
Commit: 1b2efcd7ab569896b8898e52c92ab7a676d8c45b
Parents: f8e31ec
Author: Roger Shi 
Authored: Sun Nov 6 11:50:48 2016 +0800
Committer: shaofengshi 
Committed: Sun Nov 6 15:32:53 2016 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1b2efcd7/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 2fadedb..7de0ee5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -891,8 +891,10 @@ public class CubeManager implements IRealizationProvider {
 
 if (!cubeDesc.getError().isEmpty()) {
 cube.setStatus(RealizationStatusEnum.DESCBROKEN);
-logger.warn("cube descriptor {} (for cube '{}') is broken", 
cubeDesc.getResourcePath(), cubeName);
-
+logger.error("cube descriptor {} (for cube '{}') is broken", 
cubeDesc.getResourcePath(), cubeName);
+for (String error : cubeDesc.getError()) {
+logger.error("Error: {}", error);
+}
 } else if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
 cube.setStatus(RealizationStatusEnum.DISABLED);
 logger.info("cube {} changed from DESCBROKEN to DISABLED", 
cubeName);

[01/10] kylin git commit: bug fix [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 f03f26af9 -> d4df0043e (forced update)


bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/52076862
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/52076862
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/52076862

Branch: refs/heads/yang21-cdh5.7
Commit: 52076862c098f802d6485c893663f5815f4b46f7
Parents: 44d4fe8
Author: Hongbin Ma 
Authored: Fri Nov 4 09:36:12 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 4 09:36:12 2016 +0800

--
 .../org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/52076862/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index ff729f4..3c73dc6 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -98,6 +98,10 @@ public class CubeStorageQuery implements IStorageQuery {
 
 @Override
 public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, 
TupleInfo returnTupleInfo) {
+
+//deal with participant columns in subquery join
+sqlDigest.includeSubqueryJoinParticipants();
+
 //cope with queries with no aggregations
 RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
 

[03/10] kylin git commit: minor, remove unused streaming scripts

[liyang]

minor, remove unused streaming scripts


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4f91798
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4f91798
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4f91798

Branch: refs/heads/yang21-cdh5.7
Commit: b4f917984953f1aed7a5f6c686aa930dc77bac69
Parents: ad14378
Author: Yang Li 
Authored: Sat Nov 5 08:28:02 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 5 08:28:02 2016 +0800

--
 build/bin/cleanup_streaming_files.sh | 42 ---
 build/bin/streaming_build.sh | 33 
 build/bin/streaming_check.sh | 29 -
 build/bin/streaming_fillgap.sh   | 26 ---
 build/bin/streaming_rolllog.sh   | 29 -
 5 files changed, 159 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b4f91798/build/bin/cleanup_streaming_files.sh
--
diff --git a/build/bin/cleanup_streaming_files.sh 
b/build/bin/cleanup_streaming_files.sh
deleted file mode 100644
index 9b31a4f..000
--- a/build/bin/cleanup_streaming_files.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-if [ $# != 1 ]
-then
-echo 'invalid input'
-exit -1
-fi
-
-cd $KYLIN_HOME/logs
-
-for pidfile in `find -L . -name "$1_1*"`
-do
-pidfile=`echo "$pidfile" | cut -c 3-`
-echo "pidfile:$pidfile"
-pid=`cat $pidfile`
-if [ `ps -ef | awk '{print $2}' | grep -w $pid | wc -l` = 1 ]
-then
-echo "pid:$pid still running"
-else
-echo "pid:$pid not running, try to delete files"
-echo $pidfile | xargs rm
-echo "streaming_$pidfile.log" | xargs rm
-fi
-done
-

http://git-wip-us.apache.org/repos/asf/kylin/blob/b4f91798/build/bin/streaming_build.sh
--
diff --git a/build/bin/streaming_build.sh b/build/bin/streaming_build.sh
deleted file mode 100644
index ed19036..000
--- a/build/bin/streaming_build.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-source /etc/profile
-source ~/.bash_profile
-
-CUBE=$1
-INTERVAL=$2
-DELAY=$3
-CURRENT_TIME_IN_SECOND=`date +%s`
-CURRENT_TIME=$((CURRENT_TIME_IN_SECOND * 1000))
-START=$(($CURRENT_TIME - CURRENT_TIME%INTERVAL - DELAY))
-END=$(($CURRENT_TIME - CURRENT_TIME%INTERVAL - DELAY + INTERVAL))
-
-ID="$START"_"$END"
-echo "building for ${ID}" >> ${KYLIN_HOME}/logs/build_trace.log
-sh ${KYLIN_HOME}/bin/kylin.sh streaming start ${CUBE} ${ID} -start ${START} 
-end ${END} -cube ${CUBE}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/b4f91798/build/bin/streaming_check.sh
--
diff --git a/build/bin/streaming_check.sh b/build/bin/streaming_check.sh
deleted file mode 100644
index fef0139..000
--- a/build/bin/streaming_check.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional