kylin git commit: minor, check-env.sh enhance

2016-11-11 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 371e3f4b3 -> cbd4dcc6d


minor, check-env.sh enhance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cbd4dcc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cbd4dcc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cbd4dcc6

Branch: refs/heads/master
Commit: cbd4dcc6ddf62b8caaaf753fb38c24c6727d9f92
Parents: 371e3f4
Author: Yang Li 
Authored: Sat Nov 12 09:21:13 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 12 09:24:07 2016 +0800

--
 build/bin/metastore.sh | 2 +-
 build/bin/sample.sh| 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cbd4dcc6/build/bin/metastore.sh
--
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 72afbae..d045ca6 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -26,7 +26,7 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-source ${dir}/check-env.sh
+source ${dir}/check-env.sh "if-not-yet"
 
 if [ "$1" == "backup" ]
 then

http://git-wip-us.apache.org/repos/asf/kylin/blob/cbd4dcc6/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index e420df7..cc731cf 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -19,7 +19,7 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-source ${dir}/check-env.sh
+source ${dir}/check-env.sh "if-not-yet"
 job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
 
 echo "Going to create sample tables in hive..."

kylin git commit: minor, check-env.sh enhance

2016-11-11 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang21 f5e619802 -> d34af64f6


minor, check-env.sh enhance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34af64f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34af64f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34af64f

Branch: refs/heads/yang21
Commit: d34af64f654a31236160ab298a16e08a08b8e00b
Parents: f5e6198
Author: Yang Li 
Authored: Sat Nov 12 09:21:13 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 12 09:21:13 2016 +0800

--
 build/bin/metastore.sh | 2 +-
 build/bin/sample.sh| 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d34af64f/build/bin/metastore.sh
--
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 72afbae..d045ca6 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -26,7 +26,7 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-source ${dir}/check-env.sh
+source ${dir}/check-env.sh "if-not-yet"
 
 if [ "$1" == "backup" ]
 then

http://git-wip-us.apache.org/repos/asf/kylin/blob/d34af64f/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index e420df7..cc731cf 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -19,7 +19,7 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-source ${dir}/check-env.sh
+source ${dir}/check-env.sh "if-not-yet"
 job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
 
 echo "Going to create sample tables in hive..."

kylin git commit: minor, mr code clean up

2016-11-11 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang21 b42338448 -> f5e619802


minor, mr code clean up


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f5e61980
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f5e61980
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f5e61980

Branch: refs/heads/yang21
Commit: f5e619802c54aaca03a1771c9a38af33f6adf50c
Parents: b423384
Author: Yang Li 
Authored: Fri Nov 11 22:51:23 2016 +0800
Committer: Yang Li 
Committed: Fri Nov 11 22:51:23 2016 +0800

--
 .../main/java/org/apache/kylin/engine/mr/KylinMapper.java| 6 --
 .../main/java/org/apache/kylin/engine/mr/KylinReducer.java   | 6 --
 .../java/org/apache/kylin/engine/mr/steps/CuboidJob.java | 8 +++-
 .../org/apache/kylin/engine/mr/steps/InMemCuboidJob.java | 3 ++-
 4 files changed, 13 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f5e61980/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index a527b3d..a01f7a2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-abstract public class KylinMapper extends 
Mapper {
+public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
 protected void bindCurrentConfiguration(Configuration conf) {
@@ -54,7 +54,9 @@ abstract public class KylinMapper extends Mapp
 }
 }
 
-abstract protected void doMap(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, 
InterruptedException;
+protected void doMap(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, InterruptedException {
+super.map(key, value, context);
+}
 
 @Override
 final protected void cleanup(Mapper.Context context) throws IOException, InterruptedException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f5e61980/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index 2987032..2b63ce0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-abstract public class KylinReducer extends 
Reducer {
+public class KylinReducer extends 
Reducer {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinReducer.class);
 
 protected void bindCurrentConfiguration(Configuration conf) {
@@ -53,7 +53,9 @@ abstract public class KylinReducer extends Red
 }
 }
 
-abstract protected void doReduce(KEYIN key, Iterable values, 
Reducer.Context context) throws IOException, 
InterruptedException;
+protected void doReduce(KEYIN key, Iterable values, 
Reducer.Context context) throws IOException, 
InterruptedException {
+super.reduce(key, values, context);
+}
 
 @Override
 final protected void cleanup(Reducer.Context context) throws IOException, InterruptedException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f5e61980/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index 6b0c86e..9486e60 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -21,11 +21,11 @@ package

kylin git commit: KYLIN-2174 partitoin column format visibility issue

2016-11-11 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/master a6e8c35da -> 8e22aae3a


KYLIN-2174 partitoin column format visibility issue

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8e22aae3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8e22aae3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8e22aae3

Branch: refs/heads/master
Commit: 8e22aae3a710d0b0a04b25b2b13814308bf3802f
Parents: a6e8c35
Author: luguosheng <550175...@qq.com>
Authored: Fri Nov 11 19:33:40 2016 +0800
Committer: zhongjian 
Committed: Fri Nov 11 19:40:22 2016 +0800

--
 webapp/app/js/controllers/modelEdit.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8e22aae3/webapp/app/js/controllers/modelEdit.js
--
diff --git a/webapp/app/js/controllers/modelEdit.js 
b/webapp/app/js/controllers/modelEdit.js
index 264667a..d628a0f 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -85,7 +85,7 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, 
$routeParams, $locati
 var data_type = column[0].datatype;
 if(data_type ==="bigint" ||data_type ==="int" ||data_type ==="integer"){
   $scope.isFormatVisible = false;
-  
$scope.modelsManager.selectedModel.partition_desc.partition_date_format=null;
+  
$scope.modelsManager.selectedModel.partition_desc.partition_date_format='MMdd';
   $scope.partitionColumn.hasSeparateTimeColumn=false;
   
$scope.modelsManager.selectedModel.partition_desc.partition_time_column=null;
   
$scope.modelsManager.selectedModel.partition_desc.partition_time_format=null;

kylin git commit: KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan NEW segment in cube when exception is met

2016-11-11 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 0fb0a75bd -> a6e8c35da


KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan 
NEW segment in cube when exception is met


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a6e8c35d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a6e8c35d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a6e8c35d

Branch: refs/heads/master
Commit: a6e8c35da8e2c6d944dce4c7d261a541c312b7f6
Parents: 0fb0a75
Author: Hongbin Ma 
Authored: Fri Nov 11 16:32:00 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:55:57 2016 +0800

--
 .../apache/kylin/rest/service/JobService.java   | 58 ++--
 1 file changed, 40 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a6e8c35d/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 856c754..a1aadab 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -270,25 +270,47 @@ public class JobService extends BasicService implements 
InitializingBean {
 
 checkCubeDescSignature(cube);
 DefaultChainedExecutable job;
-if (buildType == CubeBuildTypeEnum.BUILD) {
-ISource source = SourceFactory.tableSource(cube);
-SourcePartition sourcePartition = new SourcePartition(startDate, 
endDate, startOffset, endOffset, sourcePartitionOffsetStart, 
sourcePartitionOffsetEnd);
-sourcePartition = source.parsePartitionBeforeBuild(cube, 
sourcePartition);
-CubeSegment newSeg = getCubeManager().appendSegment(cube, 
sourcePartition);
-lockSegment(newSeg.getUuid());
-job = EngineFactory.createBatchCubingJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.MERGE) {
-CubeSegment newSeg = getCubeManager().mergeSegments(cube, 
startDate, endDate, startOffset, endOffset, force);
-lockSegment(newSeg.getUuid());
-job = EngineFactory.createBatchMergeJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.REFRESH) {
-CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, 
startDate, endDate, startOffset, endOffset);
-lockSegment(refreshSeg.getUuid());
-job = EngineFactory.createBatchCubingJob(refreshSeg, submitter);
-} else {
-throw new JobException("invalid build type:" + buildType);
+
+CubeSegment newSeg = null;
+try {
+if (buildType == CubeBuildTypeEnum.BUILD) {
+ISource source = SourceFactory.tableSource(cube);
+SourcePartition sourcePartition = new 
SourcePartition(startDate, endDate, startOffset, endOffset, 
sourcePartitionOffsetStart, sourcePartitionOffsetEnd);
+sourcePartition = source.parsePartitionBeforeBuild(cube, 
sourcePartition);
+newSeg = getCubeManager().appendSegment(cube, sourcePartition);
+lockSegment(newSeg.getUuid());
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.MERGE) {
+newSeg = getCubeManager().mergeSegments(cube, startDate, 
endDate, startOffset, endOffset, force);
+lockSegment(newSeg.getUuid());
+job = EngineFactory.createBatchMergeJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.REFRESH) {
+newSeg = getCubeManager().refreshSegment(cube, startDate, 
endDate, startOffset, endOffset);
+lockSegment(newSeg.getUuid());
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else {
+throw new JobException("invalid build type:" + buildType);
+}
+
+getExecutableManager().addJob(job);
+
+} catch (Exception e) {
+if (newSeg != null) {
+logger.error("Job submission might failed for NEW segment {}, 
will clean the NEW segment from cube", newSeg.getName());
+try {
+// Remove this segments
+CubeUpdate cubeBuilder = new CubeUpdate(cube);
+cubeBuilder.setToRemoveSegs(newSeg);
+getCubeManager().updateCube(cubeBuilder);
+} catch (Exception ee) {
+// swallow the exception
+

[1/6] kylin git commit: minor, JobService show possible jobs in the future [Forced Update!]

2016-11-11 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 33543a90b -> a084a869a (forced update)


minor, JobService show possible jobs in the future


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f03350
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f03350
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f03350

Branch: refs/heads/yang21-cdh5.7
Commit: 83f03350629f5c0963378c5db348bbaebd8e9aa2
Parents: 6ba39df
Author: Li Yang 
Authored: Fri Nov 11 16:08:30 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 11 16:08:51 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/JobService.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83f03350/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 5c704ba..2e547c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -95,9 +95,8 @@ public class JobService extends BasicService {
 public List listAllJobs(final String cubeName, final String 
projectName, final List statusList, final JobTimeFilterEnum 
timeFilter) {
 Calendar calendar = Calendar.getInstance();
 calendar.setTime(new Date());
-long currentTimeMillis = calendar.getTimeInMillis();
 long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
-return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, currentTimeMillis);
+return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, Long.MAX_VALUE);
 }
 
 @Deprecated

[3/6] kylin git commit: KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan NEW segment in cube when exception is met

2016-11-11 Thread mahongbin 
KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan 
NEW segment in cube when exception is met


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4233844
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4233844
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4233844

Branch: refs/heads/yang21-cdh5.7
Commit: b4233844816a722c3259109804d20a9fb8f5e954
Parents: 2e3c7ca
Author: Hongbin Ma 
Authored: Fri Nov 11 16:32:00 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:32:05 2016 +0800

--
 .../apache/kylin/rest/service/JobService.java   | 43 ++--
 1 file changed, 31 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b4233844/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 2e547c7..186e265 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -209,19 +209,38 @@ public class JobService extends BasicService {
 
 DefaultChainedExecutable job;
 
-if (buildType == CubeBuildTypeEnum.BUILD) {
-CubeSegment newSeg = getCubeManager().appendSegment(cube, 
startDate, endDate, startOffset, endOffset);
-job = EngineFactory.createBatchCubingJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.MERGE) {
-CubeSegment newSeg = getCubeManager().mergeSegments(cube, 
startDate, endDate, startOffset, endOffset, force);
-job = EngineFactory.createBatchMergeJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.REFRESH) {
-CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, 
startDate, endDate, startOffset, endOffset);
-job = EngineFactory.createBatchCubingJob(refreshSeg, submitter);
-} else {
-throw new JobException("invalid build type:" + buildType);
+CubeSegment newSeg = null;
+try {
+if (buildType == CubeBuildTypeEnum.BUILD) {
+newSeg = getCubeManager().appendSegment(cube, startDate, 
endDate, startOffset, endOffset);
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.MERGE) {
+newSeg = getCubeManager().mergeSegments(cube, startDate, 
endDate, startOffset, endOffset, force);
+job = EngineFactory.createBatchMergeJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.REFRESH) {
+newSeg = getCubeManager().refreshSegment(cube, startDate, 
endDate, startOffset, endOffset);
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else {
+throw new JobException("invalid build type:" + buildType);
+}
+getExecutableManager().addJob(job);
+
+} catch (Exception e) {
+if (newSeg != null) {
+logger.error("Job submission might failed for NEW segment {}, 
will clean the NEW segment from cube", newSeg.getName());
+try {
+// Remove this segments
+CubeUpdate cubeBuilder = new CubeUpdate(cube);
+cubeBuilder.setToRemoveSegs(newSeg);
+getCubeManager().updateCube(cubeBuilder);
+} catch (Exception ee) {
+// swallow the exception
+logger.error("Clean New segment failed, ignoring it", e);
+}
+}
+throw e;
 }
-getExecutableManager().addJob(job);
+
 JobInstance jobInstance = getSingleJobInstance(job);
 
 accessService.init(jobInstance, null);

[6/6] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-11 Thread mahongbin 
KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a084a869
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a084a869
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a084a869

Branch: refs/heads/yang21-cdh5.7
Commit: a084a869ad0726084805169658a0e47e25575960
Parents: eb6c3b0
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:45:15 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a084a869/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/a084a869/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/a084a869/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a084a869/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-11 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-11 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[2/5] kylin git commit: ignore 'defaultCatalog' in sql

2016-11-11 Thread mahongbin 
ignore 'defaultCatalog' in sql


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e3c7ca4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e3c7ca4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e3c7ca4

Branch: refs/heads/yang21-hbase1.x
Commit: 2e3c7ca43d2e5b6848a45abbffde91774fd4bf7c
Parents: 83f0335
Author: shaofengshi 
Authored: Fri Nov 11 16:12:34 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 11 16:12:34 2016 +0800

--
 .../java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java| 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e3c7ca4/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
index 8d8d971..d8c8845 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
@@ -27,6 +27,7 @@ public class KeywordDefaultDirtyHack implements 
IQueryTransformer {
 // KYLIN-2108, DEFAULT is hive default database, but a sql keyword 
too, needs quote
 sql = sql.replace("DEFAULT.", "\"DEFAULT\".");
 sql = sql.replace("default.", "\"default\".");
+sql = sql.replace("defaultCatalog.", "");
 
 return sql;
 }

[1/5] kylin git commit: minor, JobService show possible jobs in the future [Forced Update!]

2016-11-11 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x fb74c6195 -> eb6c3b064 (forced update)


minor, JobService show possible jobs in the future


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f03350
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f03350
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f03350

Branch: refs/heads/yang21-hbase1.x
Commit: 83f03350629f5c0963378c5db348bbaebd8e9aa2
Parents: 6ba39df
Author: Li Yang 
Authored: Fri Nov 11 16:08:30 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 11 16:08:51 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/JobService.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83f03350/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 5c704ba..2e547c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -95,9 +95,8 @@ public class JobService extends BasicService {
 public List listAllJobs(final String cubeName, final String 
projectName, final List statusList, final JobTimeFilterEnum 
timeFilter) {
 Calendar calendar = Calendar.getInstance();
 calendar.setTime(new Date());
-long currentTimeMillis = calendar.getTimeInMillis();
 long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
-return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, currentTimeMillis);
+return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, Long.MAX_VALUE);
 }
 
 @Deprecated

[5/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-11 Thread mahongbin 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb6c3b06
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb6c3b06
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb6c3b06

Branch: refs/heads/yang21-hbase1.x
Commit: eb6c3b06476dafdc551b88b4d956ef6cc1148b40
Parents: b423384
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:43:39 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[3/5] kylin git commit: KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan NEW segment in cube when exception is met

2016-11-11 Thread mahongbin 
KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan 
NEW segment in cube when exception is met


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4233844
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4233844
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4233844

Branch: refs/heads/yang21-hbase1.x
Commit: b4233844816a722c3259109804d20a9fb8f5e954
Parents: 2e3c7ca
Author: Hongbin Ma 
Authored: Fri Nov 11 16:32:00 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:32:05 2016 +0800

--
 .../apache/kylin/rest/service/JobService.java   | 43 ++--
 1 file changed, 31 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b4233844/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 2e547c7..186e265 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -209,19 +209,38 @@ public class JobService extends BasicService {
 
 DefaultChainedExecutable job;
 
-if (buildType == CubeBuildTypeEnum.BUILD) {
-CubeSegment newSeg = getCubeManager().appendSegment(cube, 
startDate, endDate, startOffset, endOffset);
-job = EngineFactory.createBatchCubingJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.MERGE) {
-CubeSegment newSeg = getCubeManager().mergeSegments(cube, 
startDate, endDate, startOffset, endOffset, force);
-job = EngineFactory.createBatchMergeJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.REFRESH) {
-CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, 
startDate, endDate, startOffset, endOffset);
-job = EngineFactory.createBatchCubingJob(refreshSeg, submitter);
-} else {
-throw new JobException("invalid build type:" + buildType);
+CubeSegment newSeg = null;
+try {
+if (buildType == CubeBuildTypeEnum.BUILD) {
+newSeg = getCubeManager().appendSegment(cube, startDate, 
endDate, startOffset, endOffset);
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.MERGE) {
+newSeg = getCubeManager().mergeSegments(cube, startDate, 
endDate, startOffset, endOffset, force);
+job = EngineFactory.createBatchMergeJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.REFRESH) {
+newSeg = getCubeManager().refreshSegment(cube, startDate, 
endDate, startOffset, endOffset);
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else {
+throw new JobException("invalid build type:" + buildType);
+}
+getExecutableManager().addJob(job);
+
+} catch (Exception e) {
+if (newSeg != null) {
+logger.error("Job submission might failed for NEW segment {}, 
will clean the NEW segment from cube", newSeg.getName());
+try {
+// Remove this segments
+CubeUpdate cubeBuilder = new CubeUpdate(cube);
+cubeBuilder.setToRemoveSegs(newSeg);
+getCubeManager().updateCube(cubeBuilder);
+} catch (Exception ee) {
+// swallow the exception
+logger.error("Clean New segment failed, ignoring it", e);
+}
+}
+throw e;
 }
-getExecutableManager().addJob(job);
+
 JobInstance jobInstance = getSingleJobInstance(job);
 
 accessService.init(jobInstance, null);

kylin git commit: ignore 'defaultCatalog' in sql

2016-11-11 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/yang21 83f033506 -> 2e3c7ca43


ignore 'defaultCatalog' in sql


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e3c7ca4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e3c7ca4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e3c7ca4

Branch: refs/heads/yang21
Commit: 2e3c7ca43d2e5b6848a45abbffde91774fd4bf7c
Parents: 83f0335
Author: shaofengshi 
Authored: Fri Nov 11 16:12:34 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 11 16:12:34 2016 +0800

--
 .../java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java| 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e3c7ca4/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
index 8d8d971..d8c8845 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
@@ -27,6 +27,7 @@ public class KeywordDefaultDirtyHack implements 
IQueryTransformer {
 // KYLIN-2108, DEFAULT is hive default database, but a sql keyword 
too, needs quote
 sql = sql.replace("DEFAULT.", "\"DEFAULT\".");
 sql = sql.replace("default.", "\"default\".");
+sql = sql.replace("defaultCatalog.", "");
 
 return sql;
 }

kylin git commit: ignore 'defaultCatalog' in sql

2016-11-11 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/master 4faf05876 -> 0fb0a75bd


ignore 'defaultCatalog' in sql


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fb0a75b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fb0a75b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fb0a75b

Branch: refs/heads/master
Commit: 0fb0a75bde38c7ec88d100049e08af92b205fb38
Parents: 4faf058
Author: shaofengshi 
Authored: Fri Nov 11 16:12:34 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 11 16:12:55 2016 +0800

--
 .../java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java| 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0fb0a75b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
index 8d8d971..d8c8845 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
@@ -27,6 +27,7 @@ public class KeywordDefaultDirtyHack implements 
IQueryTransformer {
 // KYLIN-2108, DEFAULT is hive default database, but a sql keyword 
too, needs quote
 sql = sql.replace("DEFAULT.", "\"DEFAULT\".");
 sql = sql.replace("default.", "\"default\".");
+sql = sql.replace("defaultCatalog.", "");
 
 return sql;
 }

kylin git commit: minor, JobService show possible jobs in the future

2016-11-11 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master e69480125 -> 4faf05876


minor, JobService show possible jobs in the future


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4faf0587
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4faf0587
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4faf0587

Branch: refs/heads/master
Commit: 4faf058761e87ac0cd8dc5386b9b39da2b52c898
Parents: e694801
Author: Li Yang 
Authored: Fri Nov 11 16:08:30 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 11 16:09:39 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/JobService.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4faf0587/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index d549fc5..856c754 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -159,9 +159,8 @@ public class JobService extends BasicService implements 
InitializingBean {
 public List listAllJobs(final String cubeName, final String 
projectName, final List statusList, final JobTimeFilterEnum 
timeFilter) {
 Calendar calendar = Calendar.getInstance();
 calendar.setTime(new Date());
-long currentTimeMillis = calendar.getTimeInMillis();
 long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
-return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, currentTimeMillis);
+return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, Long.MAX_VALUE);
 }
 
 @Deprecated

kylin git commit: minor, JobService show possible jobs in the future

2016-11-11 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang21 6ba39dfe5 -> 83f033506


minor, JobService show possible jobs in the future


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f03350
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f03350
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f03350

Branch: refs/heads/yang21
Commit: 83f03350629f5c0963378c5db348bbaebd8e9aa2
Parents: 6ba39df
Author: Li Yang 
Authored: Fri Nov 11 16:08:30 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 11 16:08:51 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/JobService.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83f03350/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 5c704ba..2e547c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -95,9 +95,8 @@ public class JobService extends BasicService {
 public List listAllJobs(final String cubeName, final String 
projectName, final List statusList, final JobTimeFilterEnum 
timeFilter) {
 Calendar calendar = Calendar.getInstance();
 calendar.setTime(new Date());
-long currentTimeMillis = calendar.getTimeInMillis();
 long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
-return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, currentTimeMillis);
+return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, Long.MAX_VALUE);
 }
 
 @Deprecated