from:"shaofengshi"

kylin git commit: KYLIN-1168 bug fix

2015-12-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 62012ad56 -> 95bc7dfa6


KYLIN-1168 bug fix

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95bc7dfa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95bc7dfa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95bc7dfa

Branch: refs/heads/1.x-staging
Commit: 95bc7dfa6d892ac01000f45152f45979b5150c7f
Parents: 62012ad
Author: shaofengshi <shaofeng...@apache.org>
Authored: Fri Dec 4 17:07:16 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Fri Dec 4 17:07:16 2015 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/CacheService.java  | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/95bc7dfa/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 398577d..38d7400 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -90,8 +90,6 @@ public class CacheService extends BasicService {
 // new data model
 getMetadataManager().reloadDataModelDesc(cacheKey);
 }
-IIDescManager.clearCache();
-CubeDescManager.clearCache();
 break;
 case ALL:
 MetadataManager.clearCache();

kylin git commit: KYLIN-930 list cube realization under each project

2015-12-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging eb692f38e -> 62012ad56


KYLIN-930 list cube realization under each project

Signed-off-by: shaofengshi <shaofeng...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/62012ad5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/62012ad5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/62012ad5

Branch: refs/heads/1.x-staging
Commit: 62012ad564b53c2fa666444977fd21cf1f042f00
Parents: eb692f3
Author: jian <jiat...@163.com>
Authored: Fri Dec 4 14:16:09 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Fri Dec 4 16:15:42 2015 +0800

--
 webapp/app/partials/projects/project_detail.html | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/62012ad5/webapp/app/partials/projects/project_detail.html
--
diff --git a/webapp/app/partials/projects/project_detail.html 
b/webapp/app/partials/projects/project_detail.html
index 35c8ef0..2644f08 100644
--- a/webapp/app/partials/projects/project_detail.html
+++ b/webapp/app/partials/projects/project_detail.html
@@ -32,7 +32,7 @@
 
 
 
-
+
 
 
 Cube Name
@@ -40,14 +40,14 @@
 
 
 
-
-{{cube}}
-Detail
+
+{{item.realization}}
+Detail
 
 
 
 
-
+

[kylin] Git Push Summary

2015-12-12 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.2-release [created] e720fa8ae

kylin git commit: KYLIN-1219 support SSO with Spring SAML, bug fix

2015-12-13 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging af889ca00 -> b331bf13e


KYLIN-1219 support SSO with Spring SAML, bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b331bf13
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b331bf13
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b331bf13

Branch: refs/heads/2.x-staging
Commit: b331bf13e9b901d03f3957de79c1d576937be22c
Parents: af889ca
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Dec 14 10:47:31 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Dec 14 10:47:57 2015 +0800

--
 build/bin/kylin.sh  | 15 ---
 server/src/main/resources/kylinSecurity.xml | 17 ++---
 2 files changed, 18 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b331bf13/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 50c5ce2..7f296cd 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -47,9 +47,14 @@ then
 #In this way we no longer need to explicitly configure hadoop/hbase 
related classpath for tomcat,
 #hbase command will do all the dirty tasks for us:
 
-
-
 spring_profile=`sh ${dir}/get-properties.sh kylin.security.profile`
+if [ -z "$spring_profile" ]
+then
+echo 'please set kylin.security.profile in kylin.properties, options 
are: testing, ldap, saml.'
+exit 1
+else
+echo "kylin.security.profile is set to $spring_profile"
+fi
 
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh
@@ -81,11 +86,7 @@ then
 -Dspring.profiles.active=${spring_profile} \
 org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar  
org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.log 
2>&1 & echo $! > ${KYLIN_HOME}/pid &
 echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh 
stop\""
-if [ "$useSandbox" = "true" ]
-then echo "Please visit http://:7070/kylin to play 
with the cubes! (Useranme: ADMIN, Password: KYLIN)"
-else
-echo "Please visit http://:7070/kylin"
-fi
+echo "Please visit http://:7070/kylin"
 echo "You can check the log at ${KYLIN_HOME}/logs/kylin.log"
 exit 0
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b331bf13/server/src/main/resources/kylinSecurity.xml
--
diff --git a/server/src/main/resources/kylinSecurity.xml 
b/server/src/main/resources/kylinSecurity.xml
index 3b2125a..522571d 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -22,12 +22,6 @@



-
-   
-   
-   
-   
-   


@@ -52,8 +46,17 @@



+
+   
+   
+   
+   
+   
+   
+   


+   



@@ -172,7 +175,7 @@



-
+

kylin git commit: KYLIN-1219 support SSO with Spring SAML, bug fix

2015-12-13 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 6da744cbe -> 2628586fe


KYLIN-1219 support SSO with Spring SAML, bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2628586f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2628586f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2628586f

Branch: refs/heads/2.0-rc
Commit: 2628586fe1aca38e74e728a71cbcf1d66eff1de3
Parents: 6da744c
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Dec 14 10:47:31 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Dec 14 10:48:59 2015 +0800

--
 build/bin/kylin.sh  | 15 ---
 server/src/main/resources/kylinSecurity.xml | 17 ++---
 2 files changed, 18 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2628586f/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 50c5ce2..7f296cd 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -47,9 +47,14 @@ then
 #In this way we no longer need to explicitly configure hadoop/hbase 
related classpath for tomcat,
 #hbase command will do all the dirty tasks for us:
 
-
-
 spring_profile=`sh ${dir}/get-properties.sh kylin.security.profile`
+if [ -z "$spring_profile" ]
+then
+echo 'please set kylin.security.profile in kylin.properties, options 
are: testing, ldap, saml.'
+exit 1
+else
+echo "kylin.security.profile is set to $spring_profile"
+fi
 
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh
@@ -81,11 +86,7 @@ then
 -Dspring.profiles.active=${spring_profile} \
 org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar  
org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.log 
2>&1 & echo $! > ${KYLIN_HOME}/pid &
 echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh 
stop\""
-if [ "$useSandbox" = "true" ]
-then echo "Please visit http://:7070/kylin to play 
with the cubes! (Useranme: ADMIN, Password: KYLIN)"
-else
-echo "Please visit http://:7070/kylin"
-fi
+echo "Please visit http://:7070/kylin"
 echo "You can check the log at ${KYLIN_HOME}/logs/kylin.log"
 exit 0
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2628586f/server/src/main/resources/kylinSecurity.xml
--
diff --git a/server/src/main/resources/kylinSecurity.xml 
b/server/src/main/resources/kylinSecurity.xml
index 3b2125a..522571d 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -22,12 +22,6 @@



-
-   
-   
-   
-   
-   


@@ -52,8 +46,17 @@



+
+   
+   
+   
+   
+   
+   
+   


+   



@@ -172,7 +175,7 @@



-
+

[kylin] Git Push Summary

2015-12-14 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.2-release-third-try [created] 57b71d1be

kylin git commit: KYLIN-1281 move “partition_date_start” to CubeDesc

2016-01-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 89b96519a -> 47ff40e20


KYLIN-1281 move âpartition_date_startâ to CubeDesc

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47ff40e2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47ff40e2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47ff40e2

Branch: refs/heads/2.0-rc
Commit: 47ff40e20b477fdfdda22e24a775bd4f3683aaa6
Parents: 89b9651
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Jan 4 15:58:11 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Jan 4 16:57:19 2016 +0800

--
 .../engine/spark/BuildCubeWithSparkTest.java|  2 +-
 .../kylin/job/BuildCubeWithEngineTest.java  |  4 ++--
 .../java/org/apache/kylin/cube/CubeManager.java | 19 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   | 21 
 .../cube/upgrade/CubeDescSignatureUpdate.java   |  1 +
 .../kylin/metadata/model/PartitionDesc.java |  3 +++
 .../cube_desc/kylin_sales_cube_desc.json|  3 ++-
 .../test_kylin_cube_with_slr_desc.json  |  3 ++-
 ...test_kylin_cube_with_slr_left_join_desc.json |  3 ++-
 .../test_kylin_cube_without_slr_desc.json   |  3 ++-
 ...t_kylin_cube_without_slr_left_join_desc.json |  3 ++-
 .../test_streaming_table_cube_desc.json |  3 ++-
 12 files changed, 49 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/47ff40e2/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
 
b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index 755b1ac..f5b9741 100644
--- 
a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -139,7 +139,7 @@ public class BuildCubeWithSparkTest {
 
 SimpleDateFormat f = new SimpleDateFormat("-MM-dd");
 f.setTimeZone(TimeZone.getTimeZone("GMT"));
-long dateStart = 
cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+long dateStart = 
cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
 long dateEnd = f.parse("2050-11-12").getTime();
 
 // this cube's start date is 0, end date is 2050111200

http://git-wip-us.apache.org/repos/asf/kylin/blob/47ff40e2/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 75a6c54..53db99a 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -254,7 +254,7 @@ public class BuildCubeWithEngineTest {
 final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
 clearSegment(cubeName);
 
-long date1 = 
cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+long date1 = 
cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
 long date2 = f.parse("2012-06-01").getTime();
 long date3 = f.parse("2022-01-01").getTime();
 long date4 = f.parse("2023-01-01").getTime();
@@ -280,7 +280,7 @@ public class BuildCubeWithEngineTest {
 
 SimpleDateFormat f = new SimpleDateFormat("-MM-dd");
 f.setTimeZone(TimeZone.getTimeZone("GMT"));
-long date1 = 
cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+long date1 = 
cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
 long date2 = f.parse("2013-01-01").getTime();
 long date3 = f.parse("2013-07-01").getTime();
 long date4 = f.parse("2022-01-01").getTime();

http://git-wip-us.apache.org/repos/asf/kylin/blob/47ff40e2/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index c5ea962..50dc166 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/mai

kylin git commit: getting mr job status with kerberos authentication

2016-01-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 604268d9f -> 7dae0cf4a


getting mr job status with kerberos authentication

Signed-off-by: fengyu <zhfen...@corp.netease.com>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7dae0cf4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7dae0cf4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7dae0cf4

Branch: refs/heads/2.0-rc
Commit: 7dae0cf4ad65f69ece49df13f85301c9470cf763
Parents: 604268d
Author: hzfen...@corp.netease.com <hzfen...@corp.netease.com>
Authored: Wed Dec 30 12:09:22 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jan 5 11:04:04 2016 +0800

--
 build/conf/kylin.properties |   2 +
 core-common/pom.xml |   4 +
 .../apache/kylin/common/KylinConfigBase.java|   4 +
 .../engine/mr/common/HadoopStatusChecker.java   |   6 +-
 .../engine/mr/common/HadoopStatusGetter.java| 104 ++-
 .../engine/mr/common/MapReduceExecutable.java   |   3 +-
 pom.xml |   6 ++
 7 files changed, 124 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7dae0cf4/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index f2170bd..d029b4c 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -82,6 +82,8 @@ kylin.hbase.region.cut.small=5
 kylin.hbase.region.cut.medium=10
 kylin.hbase.region.cut.large=50
 
+# whether get job status from resource manager with kerberos authentication
+kylin.job.status.with.kerberos=false
 
 ## kylin security configurations
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7dae0cf4/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 07a7ee1..8c7edb1 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -69,6 +69,10 @@
 commons-httpclient
 
 
+   org.apache.httpcomponents
+   httpclient
+
+
 com.google.guava
 guava
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7dae0cf4/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5968411..ee11158 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -466,6 +466,10 @@ public class KylinConfigBase implements Serializable {
 public String getHiveDatabaseForIntermediateTable() {
 return 
this.getOptional("kylin.job.hive.database.for.intermediatetable", "default");
 }
+
+public boolean isGetJobStatusWithKerberos() {
+return 
Boolean.valueOf(this.getOptional("kylin.job.status.with.kerberos", "false"));
+}
 
 public String getKylinOwner() {
 return this.getOptional("kylin.owner", "");

http://git-wip-us.apache.org/repos/asf/kylin/blob/7dae0cf4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
index 1b71b92..ef45ed1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
@@ -39,11 +39,13 @@ public class HadoopStatusChecker {
 private final String yarnUrl;
 private final String mrJobID;
 private final StringBuilder output;
+private final boolean useKerberosAuth;
 
-public HadoopStatusChecker(String yarnUrl, String mrJobID, StringBuilder 
output) {
+public HadoopStatusChecker(String yarnUrl, String mrJobID, StringBuilder 
output, boolean useKerberosAuth) {
 this.yarnUrl = yarnUrl;
 this.mrJobID = mrJobID;
 this.output = output;
+this.useKerberosAuth = useKerberosAuth;
 }
 
 public JobStepStatusEnum checkStatus() {
@@ -53,7 +55,7 @@ public class HadoopStatusChecker {
 }
 JobStepStatusEnum status = null;
 try {
-final Pair<RMAppState, FinalApplicationStatus> resul

kylin git commit: fix ci: disable export htables to tar ball by default

2016-01-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging d641ed5a6 -> 09a49554d


fix ci: disable export htables to tar ball by default


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/09a49554
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/09a49554
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/09a49554

Branch: refs/heads/1.x-staging
Commit: 09a49554d30c85f336f6a9f73b0c13023be8d058
Parents: d641ed5
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Dec 30 13:24:47 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jan 5 10:54:45 2016 +0800

--
 .../org/apache/kylin/common/persistence/HBaseConnection.java | 4 ++--
 .../test/java/org/apache/kylin/job/BuildIIWithEngineTest.java| 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/09a49554/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
index 9c86376..5b8fe54 100644
--- 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
+++ 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
@@ -64,13 +64,13 @@ public class HBaseConnection {
 }
 
 public static HConnection get(String url) {
-// find configuration
-Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
 
 HConnection connection = ConnPool.get(url);
 try {
 // I don't use DCL since recreate a connection is not a big issue.
 if (connection == null) {
+// find configuration
+Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
 connection = HConnectionManager.createConnection(conf);
 ConnPool.put(url, connection);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/09a49554/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
--
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java 
b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 4d039cf..fb9167b 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -231,7 +231,7 @@ public class BuildIIWithEngineTest {
 
 private void backup() throws Exception {
 int exitCode = cleanupOldStorage();
-if (exitCode == 0) {
+if (exitCode == 0 && 
"true".equals(System.getProperty("kylin.exportHtables"))) {
 exportHBaseData();
 }
 }

kylin git commit: getting mr job status with kerberos authentication

2016-01-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 155f1a9ce -> 72e315966


getting mr job status with kerberos authentication

Signed-off-by: fengyu <zhfen...@corp.netease.com>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/72e31596
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/72e31596
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/72e31596

Branch: refs/heads/2.x-staging
Commit: 72e31596689b6b352fbe639f9bb69705144df7cc
Parents: 155f1a9
Author: hzfen...@corp.netease.com <hzfen...@corp.netease.com>
Authored: Wed Dec 30 12:09:22 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jan 5 11:12:10 2016 +0800

--
 build/conf/kylin.properties |   3 +
 core-common/pom.xml |   4 +
 .../apache/kylin/common/KylinConfigBase.java|   4 +
 .../engine/mr/common/HadoopStatusChecker.java   |   6 +-
 .../engine/mr/common/HadoopStatusGetter.java| 104 ++-
 .../engine/mr/common/MapReduceExecutable.java   |   3 +-
 pom.xml |   6 ++
 7 files changed, 125 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/72e31596/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 0796173..836f00d 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -68,6 +68,9 @@ kylin.hbase.region.cut.large=50
 # Enable/disable ACL check for cube query
 kylin.query.security.enabled=true
 
+# whether get job status from resource manager with kerberos authentication
+kylin.job.status.with.kerberos=false
+
 ## kylin security configurations
 
 # spring security profile, options: testing, ldap, saml

http://git-wip-us.apache.org/repos/asf/kylin/blob/72e31596/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 07a7ee1..8c7edb1 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -69,6 +69,10 @@
 commons-httpclient
 
 
+   org.apache.httpcomponents
+   httpclient
+
+
 com.google.guava
 guava
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/72e31596/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5202d2d..e7c373e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -466,6 +466,10 @@ public class KylinConfigBase implements Serializable {
 public String getHiveDatabaseForIntermediateTable() {
 return 
this.getOptional("kylin.job.hive.database.for.intermediatetable", "default");
 }
+
+public boolean isGetJobStatusWithKerberos() {
+return 
Boolean.valueOf(this.getOptional("kylin.job.status.with.kerberos", "false"));
+}
 
 public String getKylinOwner() {
 return this.getOptional("kylin.owner", "");

http://git-wip-us.apache.org/repos/asf/kylin/blob/72e31596/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
index 1b71b92..ef45ed1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusChecker.java
@@ -39,11 +39,13 @@ public class HadoopStatusChecker {
 private final String yarnUrl;
 private final String mrJobID;
 private final StringBuilder output;
+private final boolean useKerberosAuth;
 
-public HadoopStatusChecker(String yarnUrl, String mrJobID, StringBuilder 
output) {
+public HadoopStatusChecker(String yarnUrl, String mrJobID, StringBuilder 
output, boolean useKerberosAuth) {
 this.yarnUrl = yarnUrl;
 this.mrJobID = mrJobID;
 this.output = output;
+this.useKerberosAuth = useKerberosAuth;
 }
 
 public JobStepStatusEnum checkStatus() {
@@ -53,7 +55,7 @@ public class HadoopStatusChecker {
 }
 JobStepStatusEnum status = null;

kylin git commit: website: update REST API doc style

2016-01-05 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 63204326a -> a8e61bb46


website: update REST API doc style

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8e61bb4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8e61bb4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8e61bb4

Branch: refs/heads/1.x-staging
Commit: a8e61bb46109ff8f2460fd9d7478136244cd0f19
Parents: 6320432
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jan 6 14:14:35 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jan 6 14:14:35 2016 +0800

--
 website/_docs/howto/howto_use_restapi.md | 23 ---
 1 file changed, 8 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a8e61bb4/website/_docs/howto/howto_use_restapi.md
--
diff --git a/website/_docs/howto/howto_use_restapi.md 
b/website/_docs/howto/howto_use_restapi.md
index fe8b419..5666f5b 100644
--- a/website/_docs/howto/howto_use_restapi.md
+++ b/website/_docs/howto/howto_use_restapi.md
@@ -144,8 +144,7 @@ Authorization:Basic {data}
  "signed":true,
  "autoIncrement":false,
  "definitelyWritable":false
-  },
-  ...
+  }
],
"results":[  
   [  
@@ -171,8 +170,7 @@ Authorization:Basic {data}
  "85.78317064220418",
  "85.78317064220418",
  "1"
-  ],
- ...
+  ]
],
"cube":"test_kylin_cube_with_slr_desc",
"affectedRowCount":0,
@@ -192,7 +190,7 @@ Authorization:Basic {data}
 * project - `required` `string` The project to load tables
 
  Response Sample
-```
+```sh
 [  
{  
   "columns":[  
@@ -257,8 +255,7 @@ Authorization:Basic {data}
   "remarks":null,
   "type_CAT":null,
   "type_NAME":null
-   },
-  ...
+   }
 ]
 ```
 
@@ -274,7 +271,7 @@ Authorization:Basic {data}
 * limit - `required` `int ` Cubes per page.
 
  Response Sample
-```
+```sh
 [  
{  
   "uuid":"1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
@@ -309,8 +306,7 @@ Authorization:Basic {data}
 * cubeName - `required` `string` Cube name.
 
  Response Sample
-
-```
+```sh
 [
 {
 "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92", 
@@ -570,8 +566,7 @@ Authorization:Basic {data}
 * modelName - `required` `string` Data model name, by default it should be the 
same as cube name.
 
  Response Sample
-
-```
+```sh
 {
 "uuid": "ff527b94-f860-44c3-8452-93b17774c647", 
 "name": "test_kylin_with_slr_model_desc", 
@@ -670,7 +665,6 @@ Authorization:Basic {data}
  "info":null,
  "run_async":true
   },
-   ...
   {  
  "interruptCmd":null,
  "name":"Load HFile to HBase Table",
@@ -719,7 +713,7 @@ Authorization:Basic {data}
 * cubeName - `required` `string` Cube name.
 
  Response Sample
-```
+```sh
 {  
"uuid":"1eaca32a-a33e-4b69-83dd-0bb8b1f8c53b",
"last_modified":1407909046305,
@@ -814,7 +808,6 @@ Authorization:Basic {data}
  "info":null,
  "run_async":true
   },
-...
   {  
  "interruptCmd":null,
  "name":"Load HFile to HBase Table",

[2/2] kylin git commit: KYLIN-1072 update documentation for REST API

2016-01-05 Thread shaofengshi

KYLIN-1072 update documentation for REST API

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/63204326
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/63204326
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/63204326

Branch: refs/heads/1.x-staging
Commit: 63204326a88fd8d1fd93dbe1cb1d6e84ba5322f2
Parents: 307e13c
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jan 6 14:01:06 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jan 6 14:01:06 2016 +0800

--
 website/_docs/howto/howto_use_restapi.md | 2108 +++--
 1 file changed, 538 insertions(+), 1570 deletions(-)
--

kylin git commit: KYLIN-1274 change "acceptPartial" default value to “false”

2016-01-05 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging a8e61bb46 -> 1322043cc


KYLIN-1274 change "acceptPartial" default value to âfalseâ

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1322043c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1322043c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1322043c

Branch: refs/heads/1.x-staging
Commit: 1322043cc5dc3ab25c9def805c17bc84a83045f0
Parents: a8e61bb
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jan 6 14:28:58 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jan 6 14:28:58 2016 +0800

--
 server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1322043c/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
--
diff --git a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java 
b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index 81ee343..4079626 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -26,7 +26,7 @@ public class SQLRequest {
 private String project;
 private Integer offset = 0;
 private Integer limit = 0;
-private boolean acceptPartial = true;
+private boolean acceptPartial = false;
 
 private Map<String, String> backdoorToggles;

kylin git commit: set 0 for all HTable cell’s timestamp

2015-12-29 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 9095b3d60 -> 2b222043d


set 0 for all HTable cellâs timestamp


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b222043
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b222043
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b222043

Branch: refs/heads/2.0-rc
Commit: 2b222043d5902766b0b0e202b062db8793cf9a95
Parents: 9095b3d
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Dec 30 10:24:14 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Dec 30 10:25:18 2015 +0800

--
 .../java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2b222043/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
index 7f40259..e56e2ea 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
@@ -33,7 +33,7 @@ public class KeyValueCreator {
 
 cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
 qBytes = Bytes.toBytes(colDesc.getQualifier());
-timestamp = System.currentTimeMillis();
+timestamp = 0; // use 0 for timestamp
 
 refIndex = colDesc.getMeasureIndex();
 refMeasures = colDesc.getMeasures();

kylin git commit: set 0 for all HTable cell’s timestamp

2015-12-29 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 7fef20524 -> fe79ed9bd


set 0 for all HTable cellâs timestamp

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fe79ed9b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fe79ed9b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fe79ed9b

Branch: refs/heads/1.x-staging
Commit: fe79ed9bdd3b695c6338594edb6474793caefbee
Parents: 7fef205
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Dec 30 10:20:34 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Dec 30 10:20:34 2015 +0800

--
 .../java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java | 2 +-
 .../java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java| 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fe79ed9b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java 
b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java
index c19e54b..f12d229 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper.java
@@ -120,7 +120,7 @@ public class CubeHFileMapper extends KylinMapper<Text, 
Text, ImmutableBytesWrita
 
 cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
 qBytes = Bytes.toBytes(colDesc.getQualifier());
-timestamp = System.currentTimeMillis();
+timestamp = 0; // use 0 for timestamp
 
 List measures = cubeDesc.getMeasures();
 String[] measureNames = getMeasureNames(cubeDesc);

http://git-wip-us.apache.org/repos/asf/kylin/blob/fe79ed9b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java 
b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
index af2850c..027c0ca 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
@@ -79,7 +79,6 @@ public class CreateHTableJob extends AbstractHadoopJob {
 // 
https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.html
 tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, 
ConstantSizeRegionSplitPolicy.class.getName());
 tableDesc.setValue(IRealizationConstants.HTableTag, 
config.getMetadataUrlPrefix());
-tableDesc.setReadOnly(true);
 
 Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
 HBaseAdmin admin = new HBaseAdmin(conf);

svn commit: r11725 - in /dev/kylin/apache-kylin-1.3-snapshot: ./ apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Wed Dec 23 02:48:47 2015
New Revision: 11725

Log:
add apache-kylin-1.3-snapshot for hbase1.1

Added:
dev/kylin/apache-kylin-1.3-snapshot/

dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
   (with props)

Added: 
dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
==
Binary file - no diff available.

Propchange: 
dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
--
svn:mime-type = application/octet-stream

svn commit: r1721480 - in /kylin/site: cn/download/index.html download/index.html feed.xml

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Wed Dec 23 01:32:50 2015
New Revision: 1721480

URL: http://svn.apache.org/viewvc?rev=1721480=rev
Log:
update link for archive site

Modified:
kylin/site/cn/download/index.html
kylin/site/download/index.html
kylin/site/feed.xml

Modified: kylin/site/cn/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/cn/download/index.html?rev=1721480=1721479=1721480=diff
==
--- kylin/site/cn/download/index.html (original)
+++ kylin/site/cn/download/index.html Wed Dec 23 01:32:50 2015
@@ -196,7 +196,7 @@
 å¯ä»¥ä»åä¸ªçæ¬æå½åææ°çå¼ååæ¯ä¸çæäºè¿å¶å
ï¼è¯·åèè¿ç¯https://kylin.apache.org/development/howto_package.html;>å¸®å©ææ¡£
 
 ä»¥åççæ¬
-Apache Kylinçæ§çæ¬å¯ä»¥ä»https://dist.apache.org/repos/dist/release/kylin/;>å½æ¡£ä¸ä¸è½½ã
+Apache Kylinçæ§çæ¬å¯ä»¥ä»https://archive.apache.org/dist/kylin/;>å½æ¡£ä¸ä¸è½½ã
 
 ODBC é©±å¨  
 Kylin ODBC é©±å¨è¦æ±é¦åå®è£Microsoft Visual C++ 2012 
Redistributableã 

Modified: kylin/site/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/download/index.html?rev=1721480=1721479=1721480=diff
==
--- kylin/site/download/index.html (original)
+++ kylin/site/download/index.html Wed Dec 23 01:32:50 2015
@@ -200,7 +200,7 @@ Also, please aware this is not a formal
 To build binary package from any version even latest development branch, 
please refer to this https://kylin.apache.org/development/howto_package.html;>guide
 
 Previous Release  
- Older releases may be found in the https://dist.apache.org/repos/dist/release/kylin/;>archives.
+ Older releases may be found in the https://archive.apache.org/dist/kylin/;>archives.
 
 ODBC Driver  
 Kylin ODBC driver requires http://www.microsoft.com/en-us/download/details.aspx?id=30679;>Microsoft 
Visual C++ 2012 Redistributable installed first. 

Modified: kylin/site/feed.xml
URL: 
http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1721480=1721479=1721480=diff
==
--- kylin/site/feed.xml (original)
+++ kylin/site/feed.xml Wed Dec 23 01:32:50 2015
@@ -19,8 +19,8 @@
 Apache Kylin Home
 http://kylin.apache.org/
 http://kylin.apache.org/feed.xml; rel="self" 
type="application/rss+xml"/>
-Tue, 22 Dec 2015 06:13:27 -0800
-Tue, 22 Dec 2015 06:13:27 -0800
+Tue, 22 Dec 2015 17:30:32 -0800
+Tue, 22 Dec 2015 17:30:32 -0800
 Jekyll v2.5.3

kylin git commit: update v1.2 binary package for hbase1.1

2015-12-22 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 0603a19e3 -> 430319316


update v1.2 binary package for hbase1.1


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/43031931
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/43031931
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/43031931

Branch: refs/heads/1.x-staging
Commit: 430319316f40874982964ed35dc8c4fd438c9000
Parents: 0603a19
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Dec 23 10:55:30 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Dec 23 10:55:30 2015 +0800

--
 website/download/index.cn.md | 4 ++--
 website/download/index.md| 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/43031931/website/download/index.cn.md
--
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index 7da5740..b5710d9 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -22,8 +22,8 @@ __äºè¿å¶å for HBase 1.1.3åæ´é«çæ¬__
 è¯·æ³¨ææ¤å®è£åéè¦HBase 
1.1.3ææ´é«çæ¬ï¼ä¹åçæ¬ä¸æä¸ä¸ªå·²ç¥çå³äºfuzzy 
keyè¿æ»¤å¨çç¼ºé·ï¼ä¼å¯¼è´Kylinæ¥è¯¢ç»æç¼ºå°è®°å½: 
[HBASE-14269](https://issues.apache.org/jira/browse/HBASE-14269)
 
æ¤å¤è¯·æ³¨æï¼è¿ä¸æ¯ä¸ä¸ªæ£å¼çåå¸çï¼æ²¡æç»è¿å®æ´çæµè¯ã
 
-  * 
[apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.2-incubating-snapshot/apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz)
-  * Git commit 
[3623dd6ff93d76141bb6a5fb623a3421ae78ca93](https://github.com/apache/kylin/commit/3623dd6ff93d76141bb6a5fb623a3421ae78ca93)
+  * 
[apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz)
+  * Git commit 
[a3b8eb04955310abec158ea30f61deb0119679d1](https://github.com/apache/kylin/commit/a3b8eb04955310abec158ea30f61deb0119679d1)
 
 
 å¯¹äºHBase 1.0ç¨æ·ï¼å»ºè®®æ¨åçº§å°1.1.3æéçº§å°0.98/0.99.
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/43031931/website/download/index.md
--
diff --git a/website/download/index.md b/website/download/index.md
index 98ae1ae..52d3413 100644
--- a/website/download/index.md
+++ b/website/download/index.md
@@ -24,8 +24,8 @@ Note the requirement of HBase version 1.1.3 (or above). There 
is a known bug in
 missing rows or lesser aggregations in Kylin query result: 
[HBASE-14269](https://issues.apache.org/jira/browse/HBASE-14269)
 Also, please aware this is not a formal release, and it is not fully tested:
 
-* 
[apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.2-incubating-snapshot/apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz)
-* Git commit 
[3623dd6ff93d76141bb6a5fb623a3421ae78ca93](https://github.com/apache/kylin/commit/3623dd6ff93d76141bb6a5fb623a3421ae78ca93)
 
+* 
[apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz)
+* Git commit 
[a3b8eb04955310abec158ea30f61deb0119679d1](https://github.com/apache/kylin/commit/a3b8eb04955310abec158ea30f61deb0119679d1)
 
 
 If you're using HBase 1.0, we suggest you to upgrade to 1.1.3+ or downgrade to 
0.98/0.99.

svn commit: r1721485 - in /kylin/site: cn/download/index.html download/index.html feed.xml

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Wed Dec 23 02:55:11 2015
New Revision: 1721485

URL: http://svn.apache.org/viewvc?rev=1721485=rev
Log:
update v1.2 binary package for hbase1.1

Modified:
kylin/site/cn/download/index.html
kylin/site/download/index.html
kylin/site/feed.xml

Modified: kylin/site/cn/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/cn/download/index.html?rev=1721485=1721484=1721485=diff
==
--- kylin/site/cn/download/index.html (original)
+++ kylin/site/cn/download/index.html Wed Dec 23 02:55:11 2015
@@ -186,8 +186,8 @@
 
æ¤å¤è¯·æ³¨æï¼è¿ä¸æ¯ä¸ä¸ªæ£å¼çåå¸çï¼æ²¡æç»è¿å®æ´çæµè¯ã
 
 
-  https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.2-incubating-snapshot/apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz;>apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz
-  Git commit https://github.com/apache/kylin/commit/3623dd6ff93d76141bb6a5fb623a3421ae78ca93;>3623dd6ff93d76141bb6a5fb623a3421ae78ca93
+  https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz;>apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
+  Git commit https://github.com/apache/kylin/commit/a3b8eb04955310abec158ea30f61deb0119679d1;>a3b8eb04955310abec158ea30f61deb0119679d1
 
 
 å¯¹äºHBase 1.0ç¨æ·ï¼å»ºè®®æ¨åçº§å°1.1.3æéçº§å°0.98/0.99.

Modified: kylin/site/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/download/index.html?rev=1721485=1721484=1721485=diff
==
--- kylin/site/download/index.html (original)
+++ kylin/site/download/index.html Wed Dec 23 02:55:11 2015
@@ -190,8 +190,8 @@ missing rows or lesser aggregations in K
 Also, please aware this is not a formal release, and it is not fully 
tested:
 
 
-  https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.2-incubating-snapshot/apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz;>apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz
-  Git commit https://github.com/apache/kylin/commit/3623dd6ff93d76141bb6a5fb623a3421ae78ca93;>3623dd6ff93d76141bb6a5fb623a3421ae78ca93
+  https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz;>apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
+  Git commit https://github.com/apache/kylin/commit/a3b8eb04955310abec158ea30f61deb0119679d1;>a3b8eb04955310abec158ea30f61deb0119679d1
 
 
 If youâre using HBase 1.0, we suggest you to upgrade to 1.1.3+ or 
downgrade to 0.98/0.99.

Modified: kylin/site/feed.xml
URL: 
http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1721485=1721484=1721485=diff
==
--- kylin/site/feed.xml (original)
+++ kylin/site/feed.xml Wed Dec 23 02:55:11 2015
@@ -19,8 +19,8 @@
 Apache Kylin Home
 http://kylin.apache.org/
 http://kylin.apache.org/feed.xml; rel="self" 
type="application/rss+xml"/>
-Tue, 22 Dec 2015 17:30:32 -0800
-Tue, 22 Dec 2015 17:30:32 -0800
+Tue, 22 Dec 2015 18:53:34 -0800
+Tue, 22 Dec 2015 18:53:34 -0800
 Jekyll v2.5.3

kylin git commit: KYLIN-1219 composite LDAP and SAML to provide authentication for both API user and human user

2015-12-23 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 99ee1b653 -> 2055efcff


KYLIN-1219 composite LDAP and SAML to provide authentication for both API user 
and human user


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2055efcf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2055efcf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2055efcf

Branch: refs/heads/2.x-staging
Commit: 2055efcff367ecad899fa8a710c0df9576325b83
Parents: 99ee1b6
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Dec 23 17:44:57 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Dec 23 17:45:25 2015 +0800

--
 .../security/KylinAuthenticationProvider.java   |  8 ++-
 server/src/main/resources/kylinSecurity.xml | 64 +++-
 2 files changed, 55 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2055efcf/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
 
b/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
index be28bdd..681fe5a 100644
--- 
a/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
+++ 
b/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
@@ -12,6 +12,7 @@ import 
org.springframework.security.authentication.AuthenticationProvider;
 import org.springframework.security.core.Authentication;
 import org.springframework.security.core.AuthenticationException;
 import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.security.core.userdetails.User;
 import org.springframework.security.core.userdetails.UserDetails;
 import org.springframework.security.saml.SAMLAuthenticationProvider;
 import org.springframework.util.Assert;
@@ -71,7 +72,12 @@ public class KylinAuthenticationProvider implements 
AuthenticationProvider {
 
 logger.debug("Authenticated user " + authed.toString());
 
-UserDetails user = (UserDetails)authed.getDetails();
+UserDetails user;
+if (authed.getDetails() instanceof  UserDetails) {
+user = (UserDetails) authed.getDetails();
+} else {
+user = new User(authentication.getName(), "skippped-ldap", 
authed.getAuthorities());
+}
 Assert.notNull(user, "The UserDetail is null.");
 
 logger.debug("User authorities :" + user.getAuthorities());

http://git-wip-us.apache.org/repos/asf/kylin/blob/2055efcf/server/src/main/resources/kylinSecurity.xml
--
diff --git a/server/src/main/resources/kylinSecurity.xml 
b/server/src/main/resources/kylinSecurity.xml
index 522571d..3bd27f3 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -53,9 +53,6 @@



-   
-   
-   



@@ -111,6 +108,9 @@


 
+   
+   
+   



@@ -118,7 +118,7 @@



-
+   

 

@@ -169,19 +169,56 @@

 

-   
+   

+   




-   
-   
-   
+
+   
+   
+   
+
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+
+   
+   
+   
+
+   
+   




+   
+
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+

kylin git commit: 增加使用kerberos认证的方式获取作业执行状态，可配置

2015-12-24 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging e72e91b47 -> e77e542c2


å¢å ä½¿ç¨kerberosè®¤è¯çæ¹å¼è·åä½ä¸æ§è¡ç¶æï¼å¯éç½®

Signed-off-by: hzfen...@corp.netease.com <hzfen...@corp.netease.com>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e77e542c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e77e542c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e77e542c

Branch: refs/heads/1.x-staging
Commit: e77e542c2ef3fa39ca558b42acbd7a322c644641
Parents: e72e91b
Author: å¯å® <hzfen...@corp.netease.com>
Authored: Mon Sep 7 20:32:23 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Fri Dec 25 10:41:47 2015 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  6 ++
 conf/kylin.properties   |  4 +
 job/pom.xml |  8 ++
 .../kylin/job/common/MapReduceExecutable.java   |  2 +-
 .../kylin/job/tools/HadoopStatusChecker.java|  8 +-
 .../kylin/job/tools/HadoopStatusGetter.java | 90 +++-
 pom.xml |  6 ++
 7 files changed, 119 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e77e542c/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index b043b2a..e0f774a 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -82,6 +82,8 @@ public class KylinConfig {
 public static final String KYLIN_JOB_REMOTE_CLI_WORKING_DIR = 
"kylin.job.remote.cli.working.dir";
 
 public static final String KYLIN_JOB_CMD_EXTRA_ARGS = 
"kylin.job.cmd.extra.args";
+
+public static final String KYLIN_GET_JOB_STATUS_WITH_KERBEROS = 
"kylin.job.status.with.kerberos";
 /**
  * Toggle to indicate whether to use hive for table flattening. Default
  * true.
@@ -419,6 +421,10 @@ public class KylinConfig {
 public String getMapReduceCmdExtraArgs() {
 return getOptional(KYLIN_JOB_CMD_EXTRA_ARGS);
 }
+
+public boolean getKylinUseKerberosAuth() {
+return Boolean.valueOf(getOptional(KYLIN_GET_JOB_STATUS_WITH_KERBEROS, 
"false"));
+}
 
 public String getOverrideHiveTableLocation(String table) {
 return getOptional(HIVE_TABLE_LOCATION_PREFIX + table.toUpperCase());

http://git-wip-us.apache.org/repos/asf/kylin/blob/e77e542c/conf/kylin.properties
--
diff --git a/conf/kylin.properties b/conf/kylin.properties
index 68db224..e196d7a 100644
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -64,6 +64,9 @@ kylin.job.concurrent.max.limit=10
 # Time interval to check hadoop job status
 kylin.job.yarn.app.rest.check.interval.seconds=10
 
+#if you should getting job status from RM with kerberos, set it true..
+kylin.job.status.with.kerberos=false
+
 # Hive database name for putting the intermediate flat tables
 kylin.job.hive.database.for.intermediatetable=default
 
@@ -142,3 +145,4 @@ kylin.monitor.ext.log.base.dir = 
/tmp/kylin_log1,/tmp/kylin_log2
 #will create external hive table to query result csv file
 #will set to kylin_query_log by default if not config here
 kylin.monitor.query.log.parse.result.table = kylin_query_log
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/e77e542c/job/pom.xml
--
diff --git a/job/pom.xml b/job/pom.xml
index c6b8b01..c62477e 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -81,6 +81,14 @@
 jackson-databind
 
 
+commons-httpclient
+commons-httpclient
+
+
+org.apache.httpcomponents
+httpclient
+
+
 com.google.guava
 guava
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e77e542c/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java 
b/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
index 2d2cfe6..a2797d7 100644
--- a/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
@@ -138,7 +138,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 return new ExecuteR

[2/2] kylin git commit: KYLIN-1230 copy cube ACL when do release

2015-12-27 Thread shaofengshi

KYLIN-1230 copy cube ACL when do release


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d7c42b3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d7c42b3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d7c42b3

Branch: refs/heads/1.x-staging
Commit: 9d7c42b3dd9febba8960ad8dac3645e799fb0747
Parents: f9de5a6
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Dec 28 11:19:36 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Dec 28 11:23:44 2015 +0800

--
 .../kylin/job/tools/CubeMigrationCLI.java   | 63 ++--
 1 file changed, 46 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9d7c42b3/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
--
diff --git a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java 
b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
index 962a4ee..44bc2c3 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hdfs.web.JsonUtil;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.*;
 import org.apache.kylin.common.util.Bytes;
@@ -72,23 +73,29 @@ public class CubeMigrationCLI {
 private static FileSystem hdfsFS;
 private static HBaseAdmin hbaseAdmin;
 
+public static final String ACL_INFO_FAMILY = "i";
+private static final String ACL_TABLE_NAME = "_acl";
+private static final String ACL_INFO_FAMILY_TYPE_COLUMN = "t";
+private static final String ACL_INFO_FAMILY_OWNER_COLUMN = "o";
+private static final String ACL_INFO_FAMILY_PARENT_COLUMN = "p";
+
 public static void main(String[] args) throws IOException, 
InterruptedException {
 
-if (args.length != 7) {
+if (args.length != 8) {
 usage();
 System.exit(1);
 }
 
-moveCube(args[0], args[1], args[2], args[3], args[4], args[5], 
args[6]);
+moveCube(args[0], args[1], args[2], args[3], args[4], args[5], 
args[6], args[7]);
 }
 
 private static void usage() {
-System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri 
dstKylinConfigUri cubeName projectName copyAclOrNot overwriteIfExists  
realExecute");
-System.out.println(" srcKylinConfigUri: The KylinConfig of the 
cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new 
home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The 
target project in the target environment.(Make sure it exist) \n" + 
"copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" 
+ "overwriteIfExists: overwrite cube if it already exists in the target 
environment. \n" + "realExecute: if false, just print the operations to take, 
if true, do the real migration. \n");
+System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri 
dstKylinConfigUri cubeName projectName copyAclOrNot purgeOrNot 
overwriteIfExists realExecute");
+System.out.println(" srcKylinConfigUri: The KylinConfig of the 
cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new 
home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The 
target project in the target environment.(Make sure it exist) \n" + 
"copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" 
+ "purgeOrNot: true or false: whether purge the cube from src server after the 
migration. \n" + "overwriteIfExists: overwrite cube if it already exists in the 
target environment. \n" + "realExecute: if false, just print the operations to 
take, if true, do the real migration. \n");
 
 }
 
-public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String 
cubeName, String projectName, String copyAcl, String overwriteIfExists, String 
realExecute) throws IOException, InterruptedException {
+public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String 
cubeName, String projectName, String copyAcl, String purgeAndDisable, String 
overwriteIfExists, String realExecute) throws IOException, InterruptedException 
{
 
 srcConfig = srcCfg;
 srcStore = ResourceStore.getStore(srcConfig);
@@ -122,10 +129,13 @@ publi

[1/2] kylin git commit: KYLIN-1014 add reference URL and support newer yarn version

2015-12-27 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging f9de5a624 -> 2533fe5d3


KYLIN-1014 add reference URL and support newer yarn version


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2533fe5d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2533fe5d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2533fe5d

Branch: refs/heads/1.x-staging
Commit: 2533fe5d3a17dd68850a2d15bd6bcb95d3b717f2
Parents: 9d7c42b
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Dec 28 11:23:11 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Dec 28 11:23:44 2015 +0800

--
 .../kylin/job/tools/HadoopStatusGetter.java | 30 
 server/pom.xml  |  5 +++-
 2 files changed, 29 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2533fe5d/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java 
b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
index 8230d48..9035ad4 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
@@ -140,6 +140,8 @@ public class HadoopStatusGetter {
 
 private static String DEFAULT_KRB5_CONFIG_LOCATION = "/etc/krb5.conf";
 private String getHttpResponseWithKerberosAuth(String url) throws 
IOException {
+
+// referred from 
https://stackoverflow.com/questions/24633380/how-do-i-authenticate-with-spnego-kerberos-and-apaches-httpclient
 String krb5ConfigPath = System.getProperty("java.security.krb5.conf");
 if (krb5ConfigPath == null) {
 krb5ConfigPath = DEFAULT_KRB5_CONFIG_LOCATION;
@@ -184,12 +186,28 @@ public class HadoopStatusGetter {
 httpget.addHeader("accept", "application/json");
 CloseableHttpResponse response = 
client.execute(httpget,context);
 String redirect = null;
-org.apache.http.Header h = response.getFirstHeader("Refresh");
+org.apache.http.Header h = response.getFirstHeader("Location");
 if (h != null) {
-String s = h.getValue();
-int cut = s.indexOf("url=");
-if (cut >= 0) {
-redirect = s.substring(cut + 4);
+redirect = h.getValue();
+if (isValidURL(redirect) == false) {
+log.info("Get invalid redirect url, skip it: " + 
redirect);
+Thread.sleep(1000l);
+continue;
+}
+} else {
+h = response.getFirstHeader("Refresh");
+if (h != null) {
+String s = h.getValue();
+int cut = s.indexOf("url=");
+if (cut >= 0) {
+redirect = s.substring(cut + 4);
+
+if (isValidURL(redirect) == false) {
+log.info("Get invalid redirect url, skip it: " 
+ redirect);
+Thread.sleep(1000l);
+continue;
+}
+}
 }
 }
 
@@ -200,6 +218,8 @@ public class HadoopStatusGetter {
 url = redirect;
 log.debug("Job " + mrJobId + " check redirect url " + url 
+ ".\n");
 } 
+} catch (InterruptedException e) {
+log.error(e.getMessage());
 } finally {
 httpget.releaseConnection();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2533fe5d/server/pom.xml
--
diff --git a/server/pom.xml b/server/pom.xml
index ee34cc6..d41fc07 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -176,7 +176,10 @@
 spring-test
 ${spring.framework.version}
 
-
+
+org.apache.httpcomponents
+httpclient
+
 
 
 org.springframework.security

kylin git commit: KYLIN-1230 copy cube ACL when do migration

2015-12-27 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc d828780ec -> 72ec98695


KYLIN-1230 copy cube ACL when do migration


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/72ec9869
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/72ec9869
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/72ec9869

Branch: refs/heads/2.0-rc
Commit: 72ec98695f60697de980dd7fb85c8519f9ce36f2
Parents: d828780
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Dec 28 11:31:06 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Dec 28 11:31:26 2015 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java| 392 +++
 1 file changed, 227 insertions(+), 165 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/72ec9869/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index a50b1eb..0e6cd75 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -18,9 +18,7 @@
 
 package org.apache.kylin.storage.hbase.util;
 
-import com.google.common.base.Preconditions;
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40,10 +38,8 @@ import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -79,22 +75,29 @@ public class CubeMigrationCLI {
 private static FileSystem hdfsFS;
 private static HBaseAdmin hbaseAdmin;
 
+public static final String ACL_INFO_FAMILY = "i";
+private static final String ACL_TABLE_NAME = "_acl";
+private static final String ACL_INFO_FAMILY_TYPE_COLUMN = "t";
+private static final String ACL_INFO_FAMILY_OWNER_COLUMN = "o";
+private static final String ACL_INFO_FAMILY_PARENT_COLUMN = "p";
+
 public static void main(String[] args) throws IOException, 
InterruptedException {
 
-if (args.length != 7) {
+if (args.length != 8) {
 usage();
 System.exit(1);
 }
 
-moveCube(args[0], args[1], args[2], args[3], args[4], args[5], 
args[6]);
+moveCube(args[0], args[1], args[2], args[3], args[4], args[5], 
args[6], args[7]);
 }
 
 private static void usage() {
-System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri 
dstKylinConfigUri cubeName projectName copyAclOrNot overwriteIfExists  
realExecute");
-System.out.println(" srcKylinConfigUri: The KylinConfig of the 
cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new 
home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The 
target project in the target environment.(Make sure it exist) \n" + 
"copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" 
+ "overwriteIfExists: overwrite cube if it already exists in the target 
environment. \n" + "realExecute: if false, just print the operations to take, 
if true, do the real migration. \n");
-  }
+System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri 
dstKylinConfigUri cubeName projectName copyAclOrNot purgeOrNot 
overwriteIfExists realExecute");
+System.out.println(" srcKylinConfigUri: The KylinConfig of the 
cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new 
home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The 
target project in the target environment.(Make sure it exist) \n" + 
"copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" 
+ "purgeOrNot: true or false: whether purge the cube from src server after the 
migration. \n" + "overwriteIfExists: overwrite cube if it al

kylin git commit: KYLIN-1230 copy cube ACL when do migration

2015-12-27 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 619af784f -> a5ea40d4d


KYLIN-1230 copy cube ACL when do migration

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a5ea40d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a5ea40d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a5ea40d4

Branch: refs/heads/2.x-staging
Commit: a5ea40d4d45954deb128662575778ebf231df49d
Parents: 619af78
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Dec 28 11:40:05 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Dec 28 11:40:05 2015 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java| 417 ++-
 1 file changed, 223 insertions(+), 194 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ea40d4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 463902f..88980f4 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -18,9 +18,7 @@
 
 package org.apache.kylin.storage.hbase.util;
 
-import com.google.common.base.Preconditions;
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40,10 +38,8 @@ import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -79,22 +75,27 @@ public class CubeMigrationCLI {
 private static FileSystem hdfsFS;
 private static HBaseAdmin hbaseAdmin;
 
+public static final String ACL_INFO_FAMILY = "i";
+private static final String ACL_TABLE_NAME = "_acl";
+private static final String ACL_INFO_FAMILY_PARENT_COLUMN = "p";
+
 public static void main(String[] args) throws IOException, 
InterruptedException {
 
-if (args.length != 7) {
+if (args.length != 8) {
 usage();
 System.exit(1);
 }
 
-moveCube(args[0], args[1], args[2], args[3], args[4], args[5], 
args[6]);
+moveCube(args[0], args[1], args[2], args[3], args[4], args[5], 
args[6], args[7]);
 }
 
 private static void usage() {
-System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri 
dstKylinConfigUri cubeName projectName copyAclOrNot overwriteIfExists  
realExecute");
-System.out.println(" srcKylinConfigUri: The KylinConfig of the 
cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new 
home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The 
target project in the target environment.(Make sure it exist) \n" + 
"copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" 
+ "overwriteIfExists: overwrite cube if it already exists in the target 
environment. \n" + "realExecute: if false, just print the operations to take, 
if true, do the real migration. \n");
+System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri 
dstKylinConfigUri cubeName projectName copyAclOrNot purgeOrNot 
overwriteIfExists realExecute");
+System.out.println(" srcKylinConfigUri: The KylinConfig of the 
cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new 
home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The 
target project in the target environment.(Make sure it exist) \n" + 
"copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" 
+ "purgeOrNot: true or false: whether purge the cube from src server after the 
migration. \n" + "overwriteIfExists: overwrite cube if it already exists in the 
target environment. \n" + "realExecute: if false, just print the operations to 
take, if true, do the real migration.

kylin git commit: KYLIN-1219 redirect user to login page if he is authenticated by SSO but doesn’t belong to LDAP group

2015-12-27 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging a5ea40d4d -> 00df6cb9a


KYLIN-1219 redirect user to login page if he is authenticated by SSO but 
doesnât belong to LDAP group


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00df6cb9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00df6cb9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00df6cb9

Branch: refs/heads/2.x-staging
Commit: 00df6cb9ab166587f27f08356b2dc4c085963e84
Parents: a5ea40d
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Dec 28 15:04:09 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Dec 28 15:06:44 2015 +0800

--
 .../kylin/rest/security/KylinAuthenticationProvider.java | 7 +++
 .../apache/kylin/rest/security/SAMLUserDetailsService.java   | 8 +++-
 server/src/main/resources/kylinSecurity.xml  | 3 +++
 3 files changed, 17 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/00df6cb9/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
 
b/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
index 681fe5a..1f147ef 100644
--- 
a/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
+++ 
b/server/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
@@ -14,6 +14,7 @@ import 
org.springframework.security.core.AuthenticationException;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.security.core.userdetails.User;
 import org.springframework.security.core.userdetails.UserDetails;
+import org.springframework.security.core.userdetails.UsernameNotFoundException;
 import org.springframework.security.saml.SAMLAuthenticationProvider;
 import org.springframework.util.Assert;
 
@@ -73,6 +74,12 @@ public class KylinAuthenticationProvider implements 
AuthenticationProvider {
 logger.debug("Authenticated user " + authed.toString());
 
 UserDetails user;
+
+if (authed.getDetails() == null) {
+//authed.setAuthenticated(false);
+throw new UsernameNotFoundException("User not found in LDAP, 
check whether he/she has been added to the groups.");
+} 
+
 if (authed.getDetails() instanceof  UserDetails) {
 user = (UserDetails) authed.getDetails();
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/00df6cb9/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java
 
b/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java
index 8d13805..c747e66 100644
--- 
a/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java
+++ 
b/server/src/main/java/org/apache/kylin/rest/security/SAMLUserDetailsService.java
@@ -25,7 +25,13 @@ public class SAMLUserDetailsService implements 
org.springframework.security.saml
 logger.debug("samlCredential.email:" + userEmail);
 final String userName = userEmail.substring(0, userEmail.indexOf("@"));
 
-UserDetails userDetails = 
ldapUserDetailsService.loadUserByUsername(userName);
+
+UserDetails userDetails = null;
+try {
+userDetails = ldapUserDetailsService.loadUserByUsername(userName);
+} catch 
(org.springframework.security.core.userdetails.UsernameNotFoundException e) {
+logger.error("User not found in LDAP, check whether he/she has 
been added to the groups.", e);
+}
 logger.debug("userDeail by search ldap with '" + userName + "' is: " + 
userDetails);
 return userDetails;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/00df6cb9/server/src/main/resources/kylinSecurity.xml
--
diff --git a/server/src/main/resources/kylinSecurity.xml 
b/server/src/main/resources/kylinSecurity.xml
index 3bd27f3..51dd601 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -175,7 +175,10 @@



+   

+   
+

[2/3] kylin git commit: [maven-release-plugin] prepare for next development iteration

2015-12-22 Thread shaofengshi

[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/390c4e7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/390c4e7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/390c4e7a

Branch: refs/heads/1.x-staging
Commit: 390c4e7ad2278563a1a224d7b4090093cd9b55c7
Parents: c2589ae
Author: shaofengshi <shaofeng...@apache.org>
Authored: Tue Dec 15 13:50:39 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Dec 15 13:50:39 2015 +0800

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 2 +-
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 2833b71..03c926c 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 1e7d5b2..2f01bde 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/cube/pom.xml
--
diff --git a/cube/pom.xml b/cube/pom.xml
index 7c4038f..b4fa9da 100644
--- a/cube/pom.xml
+++ b/cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/dictionary/pom.xml
--
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
index f0b2d13..ce86c66 100644
--- a/dictionary/pom.xml
+++ b/dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/invertedindex/pom.xml
--
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index c8d72b8..7fa3e52 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.2
+1.3-SNAPSHOT
 
 4.0.0
 kylin-invertedindex

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index fc76490..6a1f836 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@

org.apache.kylin
kylin
-   1.2
+   1.3-SNAPSHOT

 


http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/job/pom.xml
--
diff --git a/job/pom.xml b/job/pom.xml
index 23085d0..c6b8b01 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 kylin-job

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/metadata/pom.xml
--
diff --git a/metadata/pom.xml b/metadata/pom.xml
index 5ea8b13..c7c849f 100644
--- a/metadata/pom.xml
+++ b/metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/monitor/pom.xml
--
diff --git a/monitor/pom.xml b/monitor/pom.xml
index b67969d..820934f 100644
--- a/monitor/pom.xml
+++ b/monitor/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index b1c81e2..06a9043 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
 org.apache.kyli

[3/3] kylin git commit: Merge branch 'v1.2-release-fourth-try' into 1.x-staging

2015-12-22 Thread shaofengshi

Merge branch 'v1.2-release-fourth-try' into 1.x-staging


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11946ae0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11946ae0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11946ae0

Branch: refs/heads/1.x-staging
Commit: 11946ae0db1add87f52c8e1ecad9a2ff5bf5d29e
Parents: 7f99c3b 390c4e7
Author: shaofengshi <shaofeng...@apache.org>
Authored: Tue Dec 22 16:28:47 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Dec 22 16:28:47 2015 +0800

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 2 +-
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)
--

[19/50] kylin git commit: KYLIN-1119 enhance find-hive-dependency to support one common case

2015-12-22 Thread shaofengshi

KYLIN-1119 enhance find-hive-dependency to support one common case


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1cd18964
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1cd18964
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1cd18964

Branch: refs/heads/master
Commit: 1cd189649ad6d559c0d373b1e2b31c55387aa13e
Parents: 0f8fc23
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Nov 25 14:38:55 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Nov 25 14:40:07 2015 +0800

--
 bin/find-hive-dependency.sh | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1cd18964/bin/find-hive-dependency.sh
--
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
index cb5a43e..5994dda 100755
--- a/bin/find-hive-dependency.sh
+++ b/bin/find-hive-dependency.sh
@@ -44,6 +44,8 @@ then
 hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
 if [ -d "${hadoop_home}/hive-hcatalog" ]; then
   hcatalog_home=${hadoop_home}/hive-hcatalog
+elif [ -d "${hadoop_home}/hive/hcatalog" ]; then
+  hcatalog_home=${hadoop_home}/hive/hcatalog
 else 
   echo "Couldn't locate hcatalog installation, please make sure it is 
installed and set HCAT_HOME to the path."
   exit 1

[06/50] kylin git commit: KYLIN-1115 unit test with minicluster doesn't work

2015-12-22 Thread shaofengshi

KYLIN-1115 unit test with minicluster doesn't work 

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01ea9bc9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01ea9bc9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01ea9bc9

Branch: refs/heads/master
Commit: 01ea9bc9137449fc567d558fff64fb66f8cc284b
Parents: ae6f23c
Author: shaofengshi <shaofeng...@apache.org>
Authored: Thu Nov 19 21:38:38 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Thu Nov 19 21:38:38 2015 +0800

--
 .../kylin/common/persistence/HBaseResourceStore.java | 11 +++
 .../apache/kylin/common/persistence/ResourceStore.java   |  2 +-
 2 files changed, 4 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/01ea9bc9/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
index 35a62b5..23dd831 100644
--- 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
+++ 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -88,17 +89,11 @@ public class HBaseResourceStore extends ResourceStore {
 int cut = metadataUrl.indexOf('@');
 tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : 
metadataUrl.substring(0, cut);
 hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+if (!(StringUtils.isEmpty(hbaseUrl) || "hbase".equals(hbaseUrl)))
+throw new IllegalArgumentException("to use hbase storage, pls set 
'kylin.metadata.url=kylin_metadata@hbase' in kylin.properties");
 
 createHTableIfNeeded(getAllInOneTableName());
 
-//tableNameMap = new LinkedHashMap<String, String>();
-//for (Entry<String, String> entry : 
TABLE_SUFFIX_MAP.entrySet()) {
-//String pathPrefix = entry.getKey();
-//String tableName = tableNameBase + entry.getValue();
-//tableNameMap.put(pathPrefix, tableName);
-//createHTableIfNeeded(tableName);
-//}
-
 }
 
 private void createHTableIfNeeded(String tableName) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/01ea9bc9/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java 
b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 5375597..505c72a 100644
--- 
a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -61,8 +61,8 @@ abstract public class ResourceStore {
 public static final ArrayList<Class> knownImpl = 
new ArrayList<Class>();
 
 static {
-knownImpl.add(HBaseResourceStore.class);
 knownImpl.add(FileResourceStore.class);
+knownImpl.add(HBaseResourceStore.class);
 }
 
 public static ResourceStore getStore(KylinConfig kylinConfig) {

svn commit: r11696 - in /release/kylin: apache-kylin-0.7.1-incubating/ apache-kylin-0.7.2-incubating/

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Tue Dec 22 08:06:56 2015
New Revision: 11696

Log:
Remove old 0.7.x release

Removed:
release/kylin/apache-kylin-0.7.1-incubating/
release/kylin/apache-kylin-0.7.2-incubating/

svn commit: r11695 - /release/kylin/apache-kylin-1.2/

2015-12-21 Thread shaofengshi

Author: shaofengshi
Date: Tue Dec 22 07:59:33 2015
New Revision: 11695

Log:
checkin v1.2 release artifacts

Added:
release/kylin/apache-kylin-1.2/
release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz   (with props)
release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc
release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc.md5
release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc.sha1
release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.md5
release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.sha1

Added: release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz
==
Binary file - no diff available.

Propchange: release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz
--
svn:mime-type = application/octet-stream

Added: release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc
==
--- release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc (added)
+++ release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc Tue Dec 22 
07:59:33 2015
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Comment: GPGTools - https://gpgtools.org
+
+iQIcBAABCgAGBQJWb6nCAAoJEKvJ+IVvz0cgljUP/i30rmnbFCBy12zrQhluQt4O
+WEIjd+0EB8scWepeL8lOozdM48L6xjgW/f89QUXqR4gpcwRA8wV8vlPctvTM/0RV
+JWtjRCzXEJE7TGOk4CQEDvy62iqdu2G+vPF0wRvKkLSDdLXRYApYZad9kbq0BjkF
+D+ebyvMy3PW3Kzt3k7O7vsI4APJIWoCt0wzA5sY1pgv7Z99uX7LdOji05w39Gdlh
+zEOkNdpxdLEqyJXbPgl5sgu1uLdR0+7tEasoinmIlKMeswyoPgMjeo9TvoJmdJtL
+5HNiejfkYj/R3DGJmuPEmfN03MPM8N6sPnTLfHivZ5SUgN14mUDoCQgUaPzupuib
+UxqfyLwZyZ+8zaZcnrYbIjjXTpxd/qERv9hQu/RM8aNfWclDNnTSvPaHpkFO6S6H
+FG99MsOn2+pcrCd0CVdw/JdNtLe9FZk4j46Gr6ZCQya8aGGdkRPTVKi9vXvwHbPu
+PMtR1cC9MC2kTgh30dKazF/RumNEyF6hDDwDyY3z7yp299mL5bON7kyiNcR9HDc+
+WfMt1RJ5sLAiOmUtAOsASDkS7mTOKqp9IYJ8ATZYqIJTxFFqaQf9v/wQ3k9rbS7m
+sVS6WfqReVAUoPIFIk1NnWlr5JvqIrJQh/gPbj2nBbBdDLRXNPcC2bX9Fu4E2VFK
+xfMngs35P7mCtqrZ6Jf3
+=1eOR
+-END PGP SIGNATURE-

Added: release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc.md5
==
--- release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc.md5 (added)
+++ release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc.md5 Tue Dec 
22 07:59:33 2015
@@ -0,0 +1 @@
+2a75e41b8a4816fec990169e34bb825d
\ No newline at end of file

Added: release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc.sha1
==
--- release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc.sha1 (added)
+++ release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.asc.sha1 Tue Dec 
22 07:59:33 2015
@@ -0,0 +1 @@
+455d9f3ddc36373e4eb285480fc625cf3b5442d9
\ No newline at end of file

Added: release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.md5
==
--- release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.md5 (added)
+++ release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.md5 Tue Dec 22 
07:59:33 2015
@@ -0,0 +1 @@
+e1d56376f75117b97f0d6da4ad6eaee3
\ No newline at end of file

Added: release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.sha1
==
--- release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.sha1 (added)
+++ release/kylin/apache-kylin-1.2/apache-kylin-1.2-src.tar.gz.sha1 Tue Dec 22 
07:59:33 2015
@@ -0,0 +1 @@
+b8d1048825b621bdf4719f987301e4fd03763031
\ No newline at end of file

[32/50] kylin git commit: KYLIN-1168 Update CubeControllerTest and JobControllerTest

2015-12-22 Thread shaofengshi

KYLIN-1168 Update CubeControllerTest and JobControllerTest

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e568c46d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e568c46d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e568c46d

Branch: refs/heads/master
Commit: e568c46d9c390db40b3680dcfd7e68b6eeb0d5c6
Parents: 4c7bb8d
Author: shaofengshi <shaofeng...@apache.org>
Authored: Fri Dec 4 11:10:51 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Fri Dec 4 11:10:51 2015 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java| 2 --
 .../org/apache/kylin/rest/controller/CubeControllerTest.java | 4 
 .../java/org/apache/kylin/rest/controller/JobControllerTest.java | 4 
 3 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e568c46d/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 42cf206..b94ded3 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,7 +27,6 @@ import java.util.Map;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
@@ -43,7 +42,6 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
 import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.exception.NotFoundException;

http://git-wip-us.apache.org/repos/asf/kylin/blob/e568c46d/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java 
b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index bb8cb65..09ce6c5 100644
--- 
a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ 
b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -28,6 +28,7 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.rest.request.CubeRequest;
+import org.apache.kylin.rest.service.AccessService;
 import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.service.JobService;
 import org.apache.kylin.rest.service.ServiceTestBase;
@@ -52,6 +53,8 @@ public class CubeControllerTest extends ServiceTestBase {
 CubeService cubeService;
 @Autowired
 JobService jobService;
+@Autowired
+private AccessService accessService;
 
 @Before
 public void setUp() throws Exception {
@@ -59,6 +62,7 @@ public class CubeControllerTest extends ServiceTestBase {
 cubeController = new CubeController();
 cubeController.setCubeService(cubeService);
 cubeController.setJobService(jobService);
+cubeController.setAccessService(accessService);
 
 cubeDescController = new CubeDescController();
 cubeDescController.setCubeService(cubeService);

http://git-wip-us.apache.org/repos/asf/kylin/blob/e568c46d/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java 
b/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
index fccb294..24de879 100644
--- 
a/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
+++ 
b/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
@@ -33,6 +33,7 @@ import org.apache.kylin.job.dao.ExecutableDao;
 import org.apache.kylin.job.exception.PersistentException;
 import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.request.JobListRequest;
+import org.apache.kylin.rest.service.AccessService;
 import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.servic

[14/50] kylin git commit: KYLIN-1161,add get cube api

2015-12-22 Thread shaofengshi

KYLIN-1161,add get cube api


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f3214dd8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f3214dd8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f3214dd8

Branch: refs/heads/master
Commit: f3214dd8b5a3b56014584c260673f77a67ddc64e
Parents: 01a78b1
Author: Zhong 
Authored: Tue Nov 24 13:38:15 2015 +0800
Committer: Zhong 
Committed: Tue Nov 24 13:38:42 2015 +0800

--
 .../org/apache/kylin/rest/controller/CubeController.java  | 10 ++
 webapp/app/js/services/cubes.js   |  1 +
 2 files changed, 11 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f3214dd8/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 2f87c4f..9d1dfad 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -91,6 +91,16 @@ public class CubeController extends BasicController {
 return cubeService.getCubes(cubeName, projectName, (null == limit) ? 
20 : limit, offset);
 }
 
+@RequestMapping(value = "/get_cube", method = { RequestMethod.GET })
+@ResponseBody
+public CubeInstance getCube(@RequestParam(value = "cubeName", required = 
true) String cubeName) {
+CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+if (cube == null) {
+throw new InternalErrorException("Cannot find cube " + cubeName);
+}
+return cube;
+}
+
 /**
  * Get hive SQL of the cube
  *

http://git-wip-us.apache.org/repos/asf/kylin/blob/f3214dd8/webapp/app/js/services/cubes.js
--
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 3431c9a..3dfd2c5 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -19,6 +19,7 @@
 KylinApp.factory('CubeService', ['$resource', function ($resource, config) {
   return $resource(Config.service.url + 
'cubes/:cubeId/:propName/:propValue/:action', {}, {
 list: {method: 'GET', params: {}, isArray: true},
+getCube: {method: 'GET', params: {action:'get_cube'}, isArray: false},
 getSql: {method: 'GET', params: {propName: 'segs', action: 'sql'}, 
isArray: false},
 updateNotifyList: {method: 'PUT', params: {propName: 'notify_list'}, 
isArray: false},
 cost: {method: 'PUT', params: {action: 'cost'}, isArray: false},

[27/50] kylin git commit: KYLIN-1064 Restore disabled queries in KylinQueryTest.testVerifyQuery

2015-12-22 Thread shaofengshi

KYLIN-1064 Restore disabled queries in KylinQueryTest.testVerifyQuery

Signed-off-by: Li, Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2800025
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2800025
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2800025

Branch: refs/heads/master
Commit: e2800025581477992f9ab52999c56f7ead10b198
Parents: cfe9f0a
Author: lidongsjtu 
Authored: Tue Dec 1 21:11:52 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 2 11:09:21 2015 +0800

--
 .../apache/kylin/query/relnode/OLAPLimitRel.java |  7 +--
 .../resources/query/sql_verifyCount/query04.sql  | 19 +++
 .../query/sql_verifyCount/query04.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query05.sql  | 19 +++
 .../query/sql_verifyCount/query05.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query06.sql  | 19 +++
 .../query/sql_verifyCount/query06.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query07.sql  | 19 +++
 .../query/sql_verifyCount/query07.sql.disable| 19 ---
 9 files changed, 81 insertions(+), 78 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2800025/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index caf9e98..f6084ba 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -101,8 +101,11 @@ public class OLAPLimitRel extends SingleRel implements 
OLAPRel {
 
 @Override
 public EnumerableRel implementEnumerable(List inputs) {
-return new EnumerableLimit(getCluster(), 
getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
-sole(inputs), localOffset, localFetch);
+EnumerableRel input = sole(inputs);
+if (input instanceof OLAPRel) {
+((OLAPRel) input).replaceTraitSet(EnumerableConvention.INSTANCE);
+}
+return EnumerableLimit.create(input, localOffset, localFetch);
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2800025/query/src/test/resources/query/sql_verifyCount/query04.sql
--
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql 
b/query/src/test/resources/query/sql_verifyCount/query04.sql
new file mode 100644
index 000..9d3e409
--- /dev/null
+++ b/query/src/test/resources/query/sql_verifyCount/query04.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select * from test_kylin_fact limit 100

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2800025/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
--
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable 
b/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
deleted file mode 100644
index 9d3e409..000
--- a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in

[48/50] kylin git commit: KYLIN-1170 Update website and status files to TLP

2015-12-22 Thread shaofengshi

KYLIN-1170 Update website and status files to TLP


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5adddc3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5adddc3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5adddc3

Branch: refs/heads/master
Commit: d5adddc322a5e7a3b531cb9f46d4440b1e53bca3
Parents: 57b71d1
Author: shaofengshi <shaofeng...@apache.org>
Authored: Tue Dec 15 11:03:47 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Dec 15 11:04:14 2015 +0800

--
 DISCLAIMER | 12 ---
 LICENSE|  2 +-
 NOTICE |  2 +-
 pom.xml|  2 +-
 website/README.md  |  2 +-
 website/_dev/howto_release.md  | 60 +--
 website/_docs/release_notes.md | 63 -
 website/community/index.md |  2 +-
 8 files changed, 97 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/DISCLAIMER
--
diff --git a/DISCLAIMER b/DISCLAIMER
deleted file mode 100644
index 806ea66..000
--- a/DISCLAIMER
+++ /dev/null
@@ -1,12 +0,0 @@
-Apache Kylin is an effort undergoing incubation at the Apache Software
-Foundation (ASF), sponsored by the Apache Incubator PMC.
-
-Incubation is required of all newly accepted projects until a further
-review indicates that the infrastructure, communications, and decision
-making process have stabilized in a manner consistent with other
-successful ASF projects.
-
-While incubation status is not necessarily a reflection of the
-completeness or stability of the code, it does indicate that the
-project has yet to be fully endorsed by the ASF.
-

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/LICENSE
--
diff --git a/LICENSE b/LICENSE
index 11f2fd3..e2d4f7b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -205,7 +205,7 @@
 
 
 ==
-Apache Kylin (incubating) Subcomponents:
+Apache Kylin Subcomponents:
 
 The Apache Kylin project contains subcomponents with separate copyright
 notices and license terms. Your use of the source code for the these

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/NOTICE
--
diff --git a/NOTICE b/NOTICE
index f18fba3..cd0c1ba 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,4 +1,4 @@
-Apache Kylin (incubating)
+Apache Kylin
 Copyright 2014-2015 The Apache Software Foundation
 
 This product includes software developed at

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 83ddff5..709db3d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -142,7 +142,7 @@
 
scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git
 
scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git
 https://git-wip-us.apache.org/repos/asf/kylin.git
-kylin-1.1.1-incubating
+kylin-1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/website/README.md
--
diff --git a/website/README.md b/website/README.md
index 4f0f8a2..654a7b0 100644
--- a/website/README.md
+++ b/website/README.md
@@ -18,7 +18,7 @@ limitations under the License.
 -->
 
 # Apache Kylin website  
-This directory contains the source code for the Apache Kylin (incubating) 
website:
+This directory contains the source code for the Apache Kylin website:
 [http://kyin.apache.org](http://kylin.apache.org/).
 
 ## Kylin website's structure:

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/website/_dev/howto_release.md
--
diff --git a/website/_dev/howto_release.md b/website/_dev/howto_release.md
index b07bdf5..3c304a6 100644
--- a/website/_dev/howto_release.md
+++ b/website/_dev/howto_release.md
@@ -156,24 +156,24 @@ $ git clean -xn
 $ mvn clean
 
 # Do a dry run of the release:prepare step, which sets version numbers.
-$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z-incubating 
-DdevelopmentVersion=(X.Y.Z+1)-incubating-SNAPSHOT -Papache-release 
-Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE}" release:prepare 2>&1 | tee 
/tmp/prepare-dry.log
+$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z 
-DdevelopmentVersion=(X.Y.Z+1)-SNAPSHOT -Papache-release 
-Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE}" release:prepare 2>&1 | tee 
/tmp/prepare-dry

[08/50] kylin git commit: variable renaming

2015-12-22 Thread shaofengshi

variable renaming


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/320c72cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/320c72cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/320c72cc

Branch: refs/heads/master
Commit: 320c72ccb048870f5b1aa55526b6ce59cf97dbeb
Parents: b9c439e
Author: honma 
Authored: Mon Nov 23 13:21:46 2015 +0800
Committer: honma 
Committed: Mon Nov 23 13:21:46 2015 +0800

--
 bin/find-hive-dependency.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/320c72cc/bin/find-hive-dependency.sh
--
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
index ee36fc3..ea2e161 100755
--- a/bin/find-hive-dependency.sh
+++ b/bin/find-hive-dependency.sh
@@ -41,9 +41,9 @@ done
 if [ -z "$HCAT_HOME" ]
 then
 echo "HCAT_HOME not found, try to find hcatalog path from hive home"
-hive_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
-if [ -d "${hive_home}/hive-hcatalog" ]; then
-  hcatalog_home=${hive_home}/hive-hcatalog
+hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
+if [ -d "${hadoop_home}/hive-hcatalog" ]; then
+  hcatalog_home=${hadoop_home}/hive-hcatalog
 else 
   echo "Couldn't locate hcatalog installation, please make sure it is 
installed and set HCAT_HOME to the path."
   exit 1

[36/50] kylin git commit: update website, add TM mark, added 2 companies on powered by page.

2015-12-22 Thread shaofengshi

update website, add TM mark, added 2 companies on powered by page.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9457552e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9457552e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9457552e

Branch: refs/heads/master
Commit: 9457552e07555a67a5b8c0389981cb5eadd59214
Parents: 95bc7df
Author: lukehan 
Authored: Fri Dec 4 22:50:44 2015 +0800
Committer: lukehan 
Committed: Fri Dec 4 22:50:44 2015 +0800

--
 website/_data/contributors.yml | 30 --
 website/community/poweredby.md |  5 +
 website/download/index.cn.md   |  2 +-
 website/download/index.md  |  2 +-
 website/index.cn.md|  7 +++
 website/index.md   |  7 +++
 6 files changed, 37 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/_data/contributors.yml
--
diff --git a/website/_data/contributors.yml b/website/_data/contributors.yml
index dfc3d9b..17b9e38 100644
--- a/website/_data/contributors.yml
+++ b/website/_data/contributors.yml
@@ -39,7 +39,7 @@
   githubId: julianhyde
   org: Hortonworks
   role: Mentor
-- name: Luke Han (é©å¿) 
+- name: Luke Han (é©å¿) (Chair)
   apacheId: lukehan
   githubId: lukehan
   org: eBay
@@ -74,12 +74,12 @@
   apacheId: xduo
   githubId: xduo
   org: Alipay
-  role: PMC
+  role: Emeritus PMC
 - name: Ankur Bansal
   apacheId: abansal
   githubId: abansal
   org: eBay
-  role: PMC
+  role: Emeritus PMC
 - name: Xu Jiang (èæ)
   apacheId: jiangxu
   githubId: jiangxuchina
@@ -94,12 +94,30 @@
   apacheId: 
   githubId: gaodayue
   org: Meituan
-  role: committer
+  role: PMC
 - name: Hua Huang (é»æ¡¦)
   apacheId: 
   githubId: superhua
   org: MiningLAMP
+  role: PMC 
+- name: Dong Li (ææ )
+  apacheId: 
+  githubId: 
+  org: eBay
+  role: committer   
+- name: Xiaoyu Wang (çæé¨)
+  apacheId: 
+  githubId: 
+  org: JD.com
   role: committer 
-
-
+- name: Dong Li (ææ )
+  apacheId: 
+  githubId: lidongsjtu
+  org: eBay
+  role: committer 
+- name: Yerui Sun (åå¶é)
+  apacheId: sunyerui
+  githubId: 
+  org: Meituan
+  role: PMC
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/community/poweredby.md
--
diff --git a/website/community/poweredby.md b/website/community/poweredby.md
index 10d1b93..27448cc 100644
--- a/website/community/poweredby.md
+++ b/website/community/poweredby.md
@@ -25,4 +25,9 @@ __Companies & Organizations__
 * DreamSoft is using Apache Kylin as Big Data Analytics Engine for their 
customers coming from Retail, Manufacturing, Finance and others.
 * [Exponential](http://www.exponential.com)
* Using Kylin as a component in it's data platform LAMBDA to power 
inventory, campaign, behavior and demand analysis for advertising. It has been 
great at handling the size and speed constraints of old gen systems. With the 
first version already in production, we are looking forward to pushing more 
data into kylin and achieve near real time updates. 
+* [Baidu Map](http://map.baidu.com/)  (_NASDAQ: BIDU_)
+   * Apache Kylin is used for analysis Baidu Map's huge amounts of data 
about users, channels and etc, it performs quite fast with its own 
pre-calculation and index technologies along with the cubes.
+* [NetEase](http://www.163.com/)  (_NASDAQ: NTES_)
+   * Apache has been introduced as an OLAP engine for the data platform at 
NetEase, powering data analysis for various products & business with great 
performance.
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/download/index.cn.md
--
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index 81afb2e..ff4cacb 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -4,7 +4,7 @@ title: ä¸è½½
 ---
 
 __ææ°åå¸(æºä»£ç )__  
-ææ°åå¸çApache Kylinå¯ä»¥ä»ASFç½ç«ä¸è½½ï¼:
+ææ°åå¸çApache Kylinâ¢å¯ä»¥ä»ASFç½ç«ä¸è½½ï¼:
 
 * [Apache Kylin 
v1.1.1-incubating](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.1.1-incubating/)
 * [åå¸æ¥å¿](http://kylin.apache.org/docs/release_notes.html)

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/download/index.md
--
diff --git a/website/download/index.md b/website/download/index.md
index e834ccf..eed1a4b 100644
--- a/website/download/index.md
+++ b/website/download/index.md
@@ -5,7 +5,7 @@ permalink: /download/index.html
 ---
 
 __Latest

[50/50] kylin git commit: release v1.2

2015-12-22 Thread shaofengshi

release v1.2


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/13e6a434
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/13e6a434
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/13e6a434

Branch: refs/heads/master
Commit: 13e6a434a8051e7823a302ecaafb4e80cd62ed16
Parents: 5723fb6 c2589ae
Author: shaofengshi <shaofeng...@apache.org>
Authored: Tue Dec 22 16:43:33 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Dec 22 16:43:33 2015 +0800

--
 .gitignore  |   4 +
 DISCLAIMER  |  12 --
 LICENSE |  20 +-
 NOTICE  |   2 +-
 README.md   |  10 +-
 atopcalcite/pom.xml |   2 +-
 bin/check-env.sh|   0
 bin/find-hbase-dependency.sh|   0
 bin/find-hive-dependency.sh |  24 ++-
 bin/get-properties.sh   |   0
 bin/health-check.sh |   0
 bin/kylin.sh|   2 +-
 bin/metastore.sh|  11 +-
 bin/performance-monitor.sh  |   0
 bin/sample.sh   |   2 +-
 common/pom.xml  |   2 +-
 .../org/apache/kylin/common/KylinConfig.java|  22 ++-
 .../kylin/common/lock/ZookeeperJobLock.java |   2 +-
 .../common/persistence/FileResourceStore.java   |  17 +-
 .../common/persistence/HBaseConnection.java |  11 +-
 .../common/persistence/HBaseResourceStore.java  |  48 ++---
 .../kylin/common/persistence/RawResource.java   |   6 +-
 .../kylin/common/persistence/ResourceStore.java |  39 ++--
 .../kylin/common/persistence/ResourceTool.java  |  12 +-
 .../apache/kylin/common/util/DateFormat.java|   6 +-
 .../apache/kylin/common/util/HadoopUtil.java|  99 --
 .../kylin/common/util/Log4jConfigurer.java  |  55 ++
 .../common/util/AbstractKylinTestCase.java  |   5 +
 .../common/util/HBaseMetadataTestCase.java  |   7 +-
 .../common/util/HBaseMiniclusterHelper.java |  23 ++-
 conf/kylin.properties   |   2 +-
 conf/kylin_job_conf.xml |  23 +++
 cube/pom.xml|   2 +-
 .../kylin/cube/kv/AbstractRowKeyEncoder.java|  20 +-
 dictionary/pom.xml  |   2 +-
 .../apache/kylin/dict/DateStrDictionary.java|   2 +-
 .../java/org/apache/kylin/dict/Dictionary.java  |   4 +
 .../apache/kylin/dict/DictionaryGenerator.java  | 131 -
 .../apache/kylin/dict/DictionaryManager.java|   1 +
 .../kylin/dict/IDictionaryValueEnumerator.java  |  32 
 .../dict/ListDictionaryValueEnumerator.java |  50 +
 .../dict/MultipleDictionaryValueEnumerator.java |  77 
 .../kylin/dict/TableColumnValueEnumerator.java  |  75 
 .../org/apache/kylin/dict/TrieDictionary.java   |   4 +-
 .../kylin/dict/TrieDictionaryBuilder.java   |   6 +-
 .../kylin/dict/lookup/FileTableReader.java  |   2 +
 .../apache/kylin/dict/lookup/SnapshotTable.java |   3 +-
 docs/README.md  |   2 +-
 .../metadata/cube/kylin_sales_cube.json |   3 +-
 .../test_case_data/localmeta/kylin.properties   |   2 +-
 .../minicluster/b-kylin/kylin.properties|  74 ---
 .../minicluster/b-kylin/meta/kylin.properties   |  71 ---
 .../test_case_data/minicluster/kylin.properties |   4 +-
 .../test_case_data/sandbox/kylin.properties |   2 +-
 invertedindex/pom.xml   |   2 +-
 jdbc/pom.xml|   2 +-
 .../main/java/org/apache/kylin/jdbc/Driver.java |   2 +
 .../java/org/apache/kylin/jdbc/KylinClient.java | 192 +--
 .../org/apache/kylin/jdbc/KylinConnection.java  |   6 +-
 .../util/DefaultSslProtocolSocketFactory.java   |  36 ++--
 .../jdbc/util/DefaultX509TrustManager.java  |  14 +-
 .../apache/kylin/jdbc/util/Log4jConfigurer.java |  55 ++
 job/pom.xml |   2 +-
 .../apache/kylin/job/CubeMetadataUpgrade.java   |   9 +-
 .../kylin/job/common/MapReduceExecutable.java   |  18 +-
 .../org/apache/kylin/job/cube/CubingJob.java|   2 +-
 .../org/apache/kylin/job/dao/ExecutableDao.java |  12 +-
 .../kylin/job/hadoop/AbstractHadoopJob.java |  83 ++--
 .../kylin/job/hadoop/cube/BaseCuboidMapper.java |   2 +-
 .../job/hadoop/cube/MetadataCleanupJob.java |  43 -
 .../job/hadoop/invertedindex/IIBulkLoadJob.java |   2 +-
 .../hadoop/invertedindex/IICreateHFileJob.java  |   2 +-
 .../hadoop/invertedindex/IICreateHTableJob.java |   2 +-
 .../kylin/job/tools/CubeMigrationCLI.java

[44/50] kylin git commit: KYLIN-1148 fix update project issue, from nichunen <nichu...@mininglamp.com>

2015-12-22 Thread shaofengshi

KYLIN-1148 fix update project issue, from nichunen 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5f3d6ce
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5f3d6ce
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5f3d6ce

Branch: refs/heads/master
Commit: d5f3d6ce82756c00028a1c48f343137617fe370b
Parents: a39fb27
Author: jian 
Authored: Tue Dec 8 21:44:06 2015 +0800
Committer: jian 
Committed: Tue Dec 8 21:44:28 2015 +0800

--
 webapp/app/js/controllers/page.js | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d5f3d6ce/webapp/app/js/controllers/page.js
--
diff --git a/webapp/app/js/controllers/page.js 
b/webapp/app/js/controllers/page.js
index 7453d48..8382032 100644
--- a/webapp/app/js/controllers/page.js
+++ b/webapp/app/js/controllers/page.js
@@ -208,7 +208,8 @@ KylinApp.controller('PageCtrl', function ($scope, $q, 
AccessService, $modal, $lo
 var projCtrl = function ($scope, $location, $modalInstance, ProjectService, 
MessageService, projects, project, SweetAlert, ProjectModel, $cookieStore, 
$route) {
   $scope.state = {
 isEdit: false,
-oldProjName: null
+oldProjName: null,
+projectIdx: -1
   };
   $scope.isEdit = false;
   $scope.proj = {name: '', description: ''};
@@ -217,6 +218,12 @@ var projCtrl = function ($scope, $location, 
$modalInstance, ProjectService, Mess
 $scope.state.isEdit = true;
 $scope.state.oldProjName = project.name;
 $scope.proj = project;
+for (var i = 0; i < projects.length; i++){
+  if (projects[i].name === $scope.state.oldProjName){
+$scope.state.projectIdx = i;
+break;
+  }
+}
   }
 
   $scope.createOrUpdate = function () {
@@ -268,6 +275,9 @@ var projCtrl = function ($scope, $location, $modalInstance, 
ProjectService, Mess
   };
 
   $scope.cancel = function () {
+if($scope.state.isEdit){
+  projects[$scope.state.projectIdx].name = $scope.state.oldProjName;
+}
 $modalInstance.dismiss('cancel');
   };

[09/50] kylin git commit: add -L option to find command to follow symbolic links

2015-12-22 Thread shaofengshi

add -L option to find command to follow symbolic links


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/26f1665e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/26f1665e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/26f1665e

Branch: refs/heads/master
Commit: 26f1665e22ddbd67966b2f7de32571845b962993
Parents: 320c72c
Author: honma 
Authored: Mon Nov 23 13:41:17 2015 +0800
Committer: honma 
Committed: Mon Nov 23 13:41:17 2015 +0800

--
 bin/find-hive-dependency.sh | 6 +++---
 bin/sample.sh   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/26f1665e/bin/find-hive-dependency.sh
--
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
index ea2e161..cb5a43e 100755
--- a/bin/find-hive-dependency.sh
+++ b/bin/find-hive-dependency.sh
@@ -40,7 +40,7 @@ done
 # in some versions of hive hcatalog is not in hive's classpath, find it 
separately
 if [ -z "$HCAT_HOME" ]
 then
-echo "HCAT_HOME not found, try to find hcatalog path from hive home"
+echo "HCAT_HOME not found, try to find hcatalog path from hadoop home"
 hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
 if [ -d "${hadoop_home}/hive-hcatalog" ]; then
   hcatalog_home=${hadoop_home}/hive-hcatalog
@@ -53,7 +53,7 @@ else
 hcatalog_home=${HCAT_HOME}
 fi
 
-hcatalog=`find ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | 
grep -m 1 -v 'Permission denied'`
+hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 
| grep -m 1 -v 'Permission denied'`
 
 if [ -z "$hcatalog" ]
 then
@@ -62,7 +62,7 @@ then
 fi
 
 
-hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' 
-printf '%p:' | sed 's/:$//'`
+hive_lib=`find -L "$(dirname $hive_exec_path)" -name '*.jar' ! -name 
'*calcite*' -printf '%p:' | sed 's/:$//'`
 hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
 echo "hive dependency: $hive_dependency"
 export hive_dependency

http://git-wip-us.apache.org/repos/asf/kylin/blob/26f1665e/bin/sample.sh
--
diff --git a/bin/sample.sh b/bin/sample.sh
index d53393c..e02ef51 100755
--- a/bin/sample.sh
+++ b/bin/sample.sh
@@ -19,7 +19,7 @@
 
 dir=$(dirname ${0})
 source ${dir}/check-env.sh
-job_jar=`find ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
+job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
 echo "Going to create sample tables in hive..."
 cd ${KYLIN_HOME}/sample_cube/data
 hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }

[49/50] kylin git commit: [maven-release-plugin] prepare release kylin-1.2

2015-12-22 Thread shaofengshi

[maven-release-plugin] prepare release kylin-1.2


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2589aee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2589aee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2589aee

Branch: refs/heads/master
Commit: c2589aee4ac5537b460b3b02fa89cdb3a922d64e
Parents: d5adddc
Author: shaofengshi <shaofeng...@apache.org>
Authored: Tue Dec 15 13:50:03 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Dec 15 13:50:03 2015 +0800

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 2 +-
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1d9a3e1..2833b71 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 035de00..1e7d5b2 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/cube/pom.xml
--
diff --git a/cube/pom.xml b/cube/pom.xml
index 9519be5..7c4038f 100644
--- a/cube/pom.xml
+++ b/cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/dictionary/pom.xml
--
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
index 4a682e4..f0b2d13 100644
--- a/dictionary/pom.xml
+++ b/dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/invertedindex/pom.xml
--
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index 7647af2..c8d72b8 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.2-SNAPSHOT
+1.2
 
 4.0.0
 kylin-invertedindex

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 132c68c..fc76490 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@

org.apache.kylin
kylin
-   1.2-SNAPSHOT
+   1.2

 


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/job/pom.xml
--
diff --git a/job/pom.xml b/job/pom.xml
index fb677b3..23085d0 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 kylin-job

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/metadata/pom.xml
--
diff --git a/metadata/pom.xml b/metadata/pom.xml
index 259962b..5ea8b13 100644
--- a/metadata/pom.xml
+++ b/metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/monitor/pom.xml
--
diff --git a/monitor/pom.xml b/monitor/pom.xml
index cb2df81..b67969d 100644
--- a/monitor/pom.xml
+++ b/monitor/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 709db3d..b1c81e2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
 org.apache.kylin
 kylin

[31/50] kylin git commit: only cache ScanOutOfLimitException exceptions

2015-12-22 Thread shaofengshi

only cache ScanOutOfLimitException exceptions


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c7bb8de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c7bb8de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c7bb8de

Branch: refs/heads/master
Commit: 4c7bb8de94d4e910674366cf2922545443c1288c
Parents: 9e069ee
Author: honma 
Authored: Thu Dec 3 17:48:59 2015 +0800
Committer: honma 
Committed: Thu Dec 3 17:49:32 2015 +0800

--
 .../org/apache/kylin/rest/controller/QueryController.java   | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c7bb8de/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 5ac8296..c7499bd 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -44,6 +44,7 @@ import org.apache.kylin.rest.request.SaveSqlRequest;
 import org.apache.kylin.rest.response.SQLResponse;
 import org.apache.kylin.rest.service.QueryService;
 import org.apache.kylin.rest.util.QueryUtil;
+import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -191,18 +192,18 @@ public class QueryController extends BasicController {
 
 sqlResponse = new SQLResponse(null, null, 0, true, errMsg);
 
-// Access exception is bind with each user, it will not be cached
-if ((e instanceof AccessDeniedException) == false) {
+// for exception queries, only cache ScanOutOfLimitException
+if (e instanceof ScanOutOfLimitException) {
 Cache exceptionCache = 
cacheManager.getCache(EXCEPTION_QUERY_CACHE);
 exceptionCache.put(new Element(sqlRequest, sqlResponse));
 }
 }
 
 queryService.logQuery(sqlRequest, sqlResponse);
-
+
 if (sqlResponse.getIsException())
 throw new 
InternalErrorException(sqlResponse.getExceptionMessage());
-
+
 return sqlResponse;
 }

[38/50] kylin git commit: KYLIN-1205 Make HBase scan parameters configurable and allow reset of HConnection

2015-12-22 Thread shaofengshi

KYLIN-1205 Make HBase scan parameters configurable and allow reset of 
HConnection


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0c5fd3b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0c5fd3b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0c5fd3b

Branch: refs/heads/master
Commit: b0c5fd3bc7cce69dd5f0413d172fd7c7a79d9f5a
Parents: e85b243
Author: Li, Yang 
Authored: Tue Dec 8 10:43:03 2015 +0800
Committer: Li, Yang 
Committed: Tue Dec 8 10:43:03 2015 +0800

--
 .../org/apache/kylin/common/KylinConfig.java |  8 
 .../common/persistence/HBaseConnection.java  |  4 
 .../apache/kylin/rest/service/CacheService.java  |  2 ++
 .../storage/hbase/CubeSegmentTupleIterator.java  | 19 +++
 4 files changed, 29 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 9645b81..7816487 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -499,7 +499,15 @@ public class KylinConfig {
 public int getHBaseKeyValueSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", 
"10485760"));
 }
+
+public int getHBaseScanCacheRows() {
+return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
+}
 
+public int getHBaseScanMaxResultSize() {
+return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
+}
+
 public String getHbaseDefaultCompressionCodec() {
 return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC, "");
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
index dcefc24..9c86376 100644
--- 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
+++ 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
@@ -58,6 +58,10 @@ public class HBaseConnection {
 }
 });
 }
+
+public static void clearCache() {
+ConnPool.clear();
+}
 
 public static HConnection get(String url) {
 // find configuration

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 38d7400..2328592 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -21,6 +21,7 @@ package org.apache.kylin.rest.service;
 import java.io.IOException;
 import java.util.List;
 
+import org.apache.kylin.common.persistence.HBaseConnection;
 import org.apache.kylin.common.restclient.Broadcaster;
 import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.CubeManager;
@@ -101,6 +102,7 @@ public class CacheService extends BasicService {
 RealizationRegistry.clearCache();
 ProjectManager.clearCache();
 BasicService.resetOLAPDataSources();
+HBaseConnection.clearCache();
 break;
 default:
 throw new RuntimeException("invalid cacheType:" + cacheType);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
--
diff --git 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
index 9efbb79..fa4ccd7 100644
--- 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
+++ 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterList;

[45/50] kylin git commit: KYLIN-1215 minor, update website member's info on community page

2015-12-22 Thread shaofengshi

KYLIN-1215 minor, update website member's info on community page


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8d7d43d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8d7d43d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8d7d43d

Branch: refs/heads/master
Commit: b8d7d43d6e4969a0b856f9eaf33c7c499c9747f8
Parents: d5f3d6c
Author: Xiaoyu Wang 
Authored: Wed Dec 9 13:07:10 2015 +0800
Committer: Xiaoyu Wang 
Committed: Wed Dec 9 13:07:10 2015 +0800

--
 website/_data/contributors.yml | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d7d43d/website/_data/contributors.yml
--
diff --git a/website/_data/contributors.yml b/website/_data/contributors.yml
index 41145f3..d32b93f 100644
--- a/website/_data/contributors.yml
+++ b/website/_data/contributors.yml
@@ -91,28 +91,28 @@
   org: eBay
   role: PMC
 - name: Dayue Gao (é«å¤§æ)
-  apacheId: 
+  apacheId: gaodayue
   githubId: gaodayue
   org: Meituan
   role: PMC
 - name: Hua Huang (é»æ¡¦)
-  apacheId: 
+  apacheId: hhuang
   githubId: superhua
   org: MiningLAMP
-  role: PMC 
+  role: PMC
 - name: Dong Li (ææ )
-  apacheId: lidong 
+  apacheId: lidong
   githubId: lidongsjtu
   org: eBay
-  role: committer   
+  role: committer
 - name: Xiaoyu Wang (çæé¨)
   apacheId: wangxiaoyu
-  githubId: 
+  githubId: xiaowangyu
   org: JD.com
-  role: committer 
+  role: committer
 - name: Yerui Sun (åå¶é)
   apacheId: sunyerui
-  githubId: 
+  githubId: sunyerui
   org: Meituan
   role: PMC

[15/50] kylin git commit: KYLIN-1157 copy ACL when migrate cube - small fix

2015-12-22 Thread shaofengshi

KYLIN-1157 copy ACL when migrate cube - small fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a65df19
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a65df19
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a65df19

Branch: refs/heads/master
Commit: 9a65df1940cc1fea05ef146da275d07919e25fad
Parents: 7e7edff
Author: shaofengshi <shaofeng...@apache.org>
Authored: Tue Nov 24 13:39:13 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Nov 24 13:41:01 2015 +0800

--
 .../main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a65df19/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
--
diff --git a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java 
b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
index e3f7c11..ca756d1 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
@@ -76,7 +76,7 @@ public class CubeMigrationCLI {
 
 public static void main(String[] args) throws IOException, 
InterruptedException {
 
-if (args.length != 6) {
+if (args.length != 7) {
 usage();
 System.exit(1);
 }
@@ -123,7 +123,7 @@ public class CubeMigrationCLI {
 renameFoldersInHdfs(cube);
 changeHtableHost(cube);
 addCubeIntoProject(cubeName, projectName);
-if (Boolean.valueOf(copyAcl).booleanValue() == true) {
+if (Boolean.parseBoolean(copyAcl) == true) {
 copyACL(cube);
 }

[13/50] kylin git commit: KYLIN-1162 be compatible with YARN-2605

2015-12-22 Thread shaofengshi

KYLIN-1162 be compatible with YARN-2605


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01a78b1a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01a78b1a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01a78b1a

Branch: refs/heads/master
Commit: 01a78b1ab107d4b3621086862811b909ad020f62
Parents: 2a9c0cc
Author: shaofengshi <shaofeng...@apache.org>
Authored: Tue Nov 24 10:39:18 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Nov 24 10:39:51 2015 +0800

--
 .../kylin/job/tools/HadoopStatusGetter.java | 27 +---
 1 file changed, 18 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/01a78b1a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java 
b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
index 0cd4d43..e2c0aa0 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
@@ -80,16 +80,25 @@ public class HadoopStatusGetter {
 client.executeMethod(get);
 
 String redirect = null;
-Header h = get.getResponseHeader("Refresh");
+Header h = get.getResponseHeader("Location");
 if (h != null) {
-String s = h.getValue();
-int cut = s.indexOf("url=");
-if (cut >= 0) {
-redirect = s.substring(cut + 4);
-
-if (isValidURL(redirect) == false) {
-log.info("Get invalid redirect url, skip it: " + 
redirect);
-continue;
+redirect = h.getValue();
+if (isValidURL(redirect) == false) {
+log.info("Get invalid redirect url, skip it: " + 
redirect);
+continue;
+}
+} else {
+h = get.getResponseHeader("Refresh");
+if (h != null) {
+String s = h.getValue();
+int cut = s.indexOf("url=");
+if (cut >= 0) {
+redirect = s.substring(cut + 4);
+
+if (isValidURL(redirect) == false) {
+log.info("Get invalid redirect url, skip it: " 
+ redirect);
+continue;
+}
 }
 }
 }

[23/50] kylin git commit: KYLIN-1168 fix "Update data model is not allowed! Please create a new cube if needed" error

2015-12-22 Thread shaofengshi

KYLIN-1168 fix "Update data model is not allowed! Please create a new cube if 
needed" error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/04c6607f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/04c6607f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/04c6607f

Branch: refs/heads/master
Commit: 04c6607fee3775c7225d19ff55503e38238cecdb
Parents: 55c448b
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Nov 25 17:25:14 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Nov 25 17:25:44 2015 +0800

--
 .../kylin/metadata/model/DataModelDesc.java | 38 ---
 .../kylin/rest/controller/CubeController.java   | 39 +---
 .../apache/kylin/rest/service/CacheService.java |  8 ++--
 .../apache/kylin/rest/service/CubeService.java  |  2 +-
 4 files changed, 31 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/04c6607f/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java 
b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 88fafdf..f8dc54d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -220,44 +220,6 @@ public class DataModelDesc extends RootPersistentEntity {
 }
 }
 
-/**
- * Check whether two data model are compatible or not. Compatible means
- * having the same structure. Tow models could be compatible even they
- * have different UUID or last modified time.
- * @param that model to compare with
- * @return true if compatible, false otherwise.
- */
-public boolean compatibleWith(DataModelDesc that) {
-if (this == that)
-return true;
-
-if (that == null)
-return false;
-
-try {
-String thisRepr = excludeHeaderInfo(this);
-String thatRepr = excludeHeaderInfo(that);
-return StringUtils.equals(thisRepr, thatRepr);
-
-} catch (IOException e) {
-logger.error("Failed to serialize DataModelDesc to string", e);
-return false;
-}
-}
-
-private String excludeHeaderInfo(DataModelDesc modelDesc) throws 
IOException {
-// make a copy
-String repr = JsonUtil.writeValueAsString(modelDesc);
-DataModelDesc copy = JsonUtil.readValue(repr, DataModelDesc.class);
-
-copy.setUuid(null);
-copy.setLastModified(0);
-copy.setCapacity(RealizationCapacity.MEDIUM);
-copy.setFilterCondition(null);
-copy.setPartitionDesc(null);
-return JsonUtil.writeValueAsString(copy);
-}
-
 @Override
 public String toString() {
 return "DataModelDesc [name=" + name + "]";

http://git-wip-us.apache.org/repos/asf/kylin/blob/04c6607f/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 9d1dfad..8565d5e 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -359,8 +359,8 @@ public class CubeController extends BasicController {
 }
 
 MetadataManager metadataManager = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
-// KYLIN-958: disallow data model structure change
 DataModelDesc modelDesc = null;
+DataModelDesc oldModelDesc = null;
 if (StringUtils.isNotEmpty(cubeRequest.getModelDescData())) {
 modelDesc = deserializeDataModelDesc(cubeRequest);
 if (modelDesc == null) {
@@ -373,15 +373,11 @@ public class CubeController extends BasicController {
 return errorRequest(cubeRequest, "CubeDesc.model_name " + 
desc.getModelName() + " not consistent with model " + modeName);
 }
 
-DataModelDesc oldModelDesc = 
metadataManager.getDataModelDesc(modeName);
+oldModelDesc = metadataManager.getDataModelDesc(modeName);
 if (oldModelDesc == null) {
 return errorRequest(cubeRequest, "Data model " + modeName + " 
not found");
 }
 
-if (!modelDesc.compatibleWith(oldModelDesc)) {
-return erro

[30/50] kylin git commit: KYLIN-1168 add UT

2015-12-22 Thread shaofengshi

KYLIN-1168 add UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e069eed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e069eed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e069eed

Branch: refs/heads/master
Commit: 9e069eed066f356909ac34a570707219d95ecb5e
Parents: 316bc04
Author: shaofengshi <shaofeng...@apache.org>
Authored: Thu Dec 3 14:54:22 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Thu Dec 3 14:57:19 2015 +0800

--
 .../kylin/rest/controller/CubeController.java   |  50 +++--
 .../kylin/rest/controller/JobController.java|   2 +-
 .../apache/kylin/rest/service/BasicService.java |  10 -
 .../apache/kylin/rest/service/CacheService.java |  24 ++-
 .../apache/kylin/rest/service/CubeService.java  |  42 +
 .../kylin/rest/service/CacheServiceTest.java| 186 +++
 .../kylin/rest/service/CubeServiceTest.java |   4 -
 .../kylin/rest/service/JobServiceTest.java  |   1 -
 .../kylin/rest/service/QueryServiceTest.java|   1 -
 9 files changed, 219 insertions(+), 101 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9e069eed/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f27f312..42cf206 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -41,6 +41,8 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
 import org.apache.kylin.rest.exception.InternalErrorException;
@@ -49,6 +51,8 @@ import org.apache.kylin.rest.request.CubeRequest;
 import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.response.GeneralResponse;
 import org.apache.kylin.rest.response.HBaseResponse;
+import org.apache.kylin.rest.security.AclPermission;
+import org.apache.kylin.rest.service.AccessService;
 import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.service.JobService;
 import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
@@ -83,6 +87,9 @@ public class CubeController extends BasicController {
 private CubeService cubeService;
 
 @Autowired
+private AccessService accessService;
+
+@Autowired
 private JobService jobService;
 
 @RequestMapping(value = "", method = { RequestMethod.GET })
@@ -278,6 +285,7 @@ public class CubeController extends BasicController {
 
 try {
 cubeService.deleteCube(cube);
+accessService.clean(cube, true);
 } catch (Exception e) {
 logger.error(e.getLocalizedMessage(), e);
 throw new InternalErrorException("Failed to delete cube. " + " 
Caused by: " + e.getMessage(), e);
@@ -285,21 +293,25 @@ public class CubeController extends BasicController {
 }
 
 /**
- * Get available table list of the input database
+ * Create a new Cube
  *
- * @return Table metadata array
+ * @return cubeRequest cube change request
  * @throws IOException
  */
 @RequestMapping(value = "", method = { RequestMethod.POST })
 @ResponseBody
 public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) {
 //Update Model
-MetadataManager metaManager = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+MetadataManager metaManager = 
MetadataManager.getInstance(cubeService.getConfig());
 DataModelDesc modelDesc = deserializeDataModelDesc(cubeRequest);
-if (modelDesc == null || StringUtils.isEmpty(modelDesc.getName())) {
-return cubeRequest;
+if (modelDesc == null) {
+return errorRequest(cubeRequest, "Missing ModelDesc data in the 
request.");
 }
 
+if (StringUtils.isEmpty(modelDesc.getName())) {
+return errorRequest(cubeRequest, "Missing modelName.");
+}
+
 try {
 DataModelDesc existingModel = 
metaManager.getDataModelDesc(modelDesc.getName());
 if (existingModel == null) {

[39/50] kylin git commit: KYLIN-1128 clone cube metadata

2015-12-22 Thread shaofengshi

KYLIN-1128 clone cube metadata


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ad77cca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ad77cca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ad77cca

Branch: refs/heads/master
Commit: 9ad77cca832dfabb562b8e12d5e28e9e505e245d
Parents: d18b73a
Author: jian 
Authored: Fri Dec 4 12:48:38 2015 +0800
Committer: jian 
Committed: Tue Dec 8 11:03:05 2015 +0800

--
 .../apache/kylin/metadata/MetadataManager.java  | 21 
 .../kylin/rest/controller/CubeController.java   | 15 ++
 webapp/app/js/controllers/cubes.js  |  4 +---
 webapp/app/partials/cubes/cube_clone.html   | 13 
 4 files changed, 33 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ad77cca/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index b540588..800ea44 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -40,6 +40,7 @@ import 
org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.ProjectManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -341,6 +342,26 @@ public class MetadataManager {
 return saveDataModelDesc(desc);
 }
 
+// sync on update
+public DataModelDesc dropModel(DataModelDesc desc) throws IOException {
+logger.info("Dropping model '" + desc.getName() + "'");
+ResourceStore store = getStore();
+if (desc != null)
+store.deleteResource(desc.getResourcePath());
+// clean model cache
+this.afterModelDropped(desc);
+return desc;
+}
+
+private void afterModelDropped(DataModelDesc desc) {
+removeModelCache(desc);
+}
+
+private void removeModelCache(DataModelDesc desc) {
+dataModelDescMap.remove(desc.getName());
+}
+
+
 private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) 
throws IOException {
 dataModelDesc.init(this.getAllTablesMap());
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ad77cca/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b35a35c..95169da 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
@@ -260,7 +261,6 @@ public class CubeController extends BasicController {
 @ResponseBody
 public CubeInstance cloneCube(@PathVariable String cubeName,@RequestBody 
CubeRequest cubeRequest) {
 String targetCubeName = cubeRequest.getCubeName();
-String targetModelName = cubeRequest.getModelDescData();
 String project = cubeRequest.getProject();
 
 CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -275,11 +275,13 @@ public class CubeController extends BasicController {
 
 DataModelDesc modelDesc = metaManager.getDataModelDesc(modelName);
 
-modelDesc.setName(targetModelName);
+//model name same as cube
+modelDesc.setName(targetCubeName);
 modelDesc.setLastModified(0);
 modelDesc.setUuid(UUID.randomUUID().toString());
+DataModelDesc newModel = null;
 try {
-metaManager.createDataModelDesc(modelDesc);
+newModel = metaManager.createDataModelDesc(modelDesc);
 } catch (IOException e) {
 throw new InternalErrorException("failed to clone 
DataModelDesc",e);
 }
@@ -287,11 +289,16 @@ public class CubeController extends BasicController {
 cubeDesc.setName(targetCubeName);
 cubeDesc.setLastModified(0);
 cubeDesc.setUuid(UUID.randomUUID().toString());
-

[24/50] kylin git commit: KYLIN-1168 fix UT

2015-12-22 Thread shaofengshi

KYLIN-1168 fix UT

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f5b1561
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f5b1561
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f5b1561

Branch: refs/heads/master
Commit: 0f5b1561712b290a00c100e00faf97f430742698
Parents: 04c6607
Author: shaofengshi <shaofeng...@apache.org>
Authored: Thu Nov 26 11:27:56 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Thu Nov 26 11:27:56 2015 +0800

--
 .../org/apache/kylin/rest/controller/CubeController.java |  1 +
 .../java/org/apache/kylin/rest/service/CacheService.java |  7 +--
 .../org/apache/kylin/rest/service/CacheServiceTest.java  | 11 ---
 3 files changed, 10 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5b1561/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8565d5e..f27f312 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -414,6 +414,7 @@ public class CubeController extends BasicController {
 if (updateModelSuccess == true && updateCubeSuccess == false ) {
 // recover data model
 try {
+oldModelDesc.setLastModified(modelDesc.getLastModified());
 metadataManager.updateDataModelDesc(oldModelDesc);
 } catch (IOException e) {
 logger.error("Failed to recover data model desc:" + 
e.getLocalizedMessage(), e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5b1561/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index c953b7e..7a3238c 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -59,8 +59,10 @@ public class CacheService extends BasicService {
 cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
 break;
 case CUBE_DESC:
-String modelName = 
getCubeDescManager().getCubeDesc(cacheKey).getModelName();
-getMetadataManager().reloadDataModelDesc(modelName);
+if (getCubeDescManager().getCubeDesc(cacheKey) != null) {
+String modelName = 
getCubeDescManager().getCubeDesc(cacheKey).getModelName();
+getMetadataManager().reloadDataModelDesc(modelName);
+}
 getCubeDescManager().reloadCubeDesc(cacheKey);
 IIDescManager.clearCache();
 CubeDescManager.clearCache();
@@ -143,4 +145,5 @@ public class CacheService extends BasicService {
 throw new RuntimeException("error " + log, e);
 }
 }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5b1561/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java 
b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 0436dc9..44755d4 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -47,17 +47,13 @@ import 
org.apache.kylin.rest.broadcaster.BroadcasterReceiveServlet;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
 import org.slf4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
 
 /**
- * Created by qianzhou on 1/16/15.
  */
-
+@Ignore ("Not working for some time")
 public class CacheServiceTest extends LocalFileMetadataTestCase {
 
 private static Server server;
@@ -281,6 +277,7 @@ public class CacheServiceTest extends 
LocalFileMetadataTestCase {
 }
 
 @Test
+//@Ignore ("disable this as only data model change will not trigger a 
broad cast event,

[28/50] kylin git commit: KYLIN-1190 Make memory budget per query configurable

2015-12-22 Thread shaofengshi

KYLIN-1190 Make memory budget per query configurable

Signed-off-by: Li, Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7888c87f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7888c87f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7888c87f

Branch: refs/heads/master
Commit: 7888c87fa190e950227eee80d59f8a108b93a18f
Parents: e280002
Author: lidongsjtu 
Authored: Tue Dec 1 16:32:17 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 2 11:11:59 2015 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfig.java|  4 
 .../org/apache/kylin/storage/hbase/CubeStorageEngine.java | 10 +++---
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7888c87f/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index a8f7c5d..9645b81 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -492,6 +492,10 @@ public class KylinConfig {
 return 
Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true"));
 }
 
+public long getQueryMemBudget() {
+return Long.parseLong(this.getOptional("kylin.query.mem.budget", 
String.valueOf(3L * 1024 * 1024 * 1024)));
+}
+
 public int getHBaseKeyValueSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", 
"10485760"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7888c87f/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
--
diff --git 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index ed12781..bbf088e 100644
--- 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -76,7 +76,6 @@ public class CubeStorageEngine implements IStorageEngine {
 private static final Logger logger = 
LoggerFactory.getLogger(CubeStorageEngine.class);
 
 private static final int MERGE_KEYRANGE_THRESHOLD = 100;
-private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; 
// 3G
 
 private final CubeInstance cubeInstance;
 private final CubeDesc cubeDesc;
@@ -643,8 +642,13 @@ public class CubeStorageEngine implements IStorageEngine {
 }
 }
 
-long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
-context.setThreshold((int) rowEst);
+long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / 
rowSizeEst;
+if (rowEst > 0) {
+logger.info("Memory budget is set to: " + rowEst);
+context.setThreshold((int) rowEst);
+} else {
+logger.info("Memory budget is not set.");
+}
 }
 
 private void setLimit(TupleFilter filter, StorageContext context) {

svn commit: r1721410 - in /kylin/site: cn/download/index.html community/index.html development/howto_package.html development/howto_release.html download/index.html feed.xml

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Tue Dec 22 14:20:24 2015
New Revision: 1721410

URL: http://svn.apache.org/viewvc?rev=1721410=rev
Log:
update with v1.2 release

Modified:
kylin/site/cn/download/index.html
kylin/site/community/index.html
kylin/site/development/howto_package.html
kylin/site/development/howto_release.html
kylin/site/download/index.html
kylin/site/feed.xml

Modified: kylin/site/cn/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/cn/download/index.html?rev=1721410=1721409=1721410=diff
==
--- kylin/site/cn/download/index.html (original)
+++ kylin/site/cn/download/index.html Tue Dec 22 14:20:24 2015
@@ -166,17 +166,17 @@
 ææ°åå¸çApache Kylinâ¢å¯ä»¥ä»ASFç½ç«ä¸è½½ï¼:
 
 
-  http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.1.1-incubating/;>Apache
 Kylin v1.1.1-incubating
+  http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.2/;>Apache 
Kylin v1.2
   http://kylin.apache.org/docs/release_notes.html;>åå¸æ¥å¿
-  Git æ ç¾: https://github.com/apache/kylin/tree/kylin-1.1.1-incubating;>kylin-1.1.1-incubating
-  Git Commit: https://github.com/apache/kylin/commit/6a9499b4d0fabb54211a8a536c2e18d3fe8b4a5d;>6a9499b4d0fabb54211a8a536c2e18d3fe8b4a5d
+  Git æ ç¾: https://github.com/apache/kylin/tree/kylin-1.2;>kylin-1.2
+  Git Commit: https://github.com/apache/kylin/commit/c2589aee4ac5537b460b3b02fa89cdb3a922d64e;>c2589aee4ac5537b460b3b02fa89cdb3a922d64e
 
 
 äºè¿å¶å for HBase 0.98/0.99
 ä¸ºæ¹ä¾¿ä½¿ç¨ï¼æä»¬æä¾é¢æåçäºè¿å¶å®è£åï¼
 
 
-  https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.1.1-incubating/apache-kylin-1.1.1-incubating-bin.tar.gz;>apache-kylin-1.1.1-incubating-bin.tar.gz
+  https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.2/apache-kylin-1.2-bin.tar.gz;>apache-kylin-1.2-bin.tar.gz
   http://kylin.apache.org/docs/install;>å®è£å¸®å©
 
 
@@ -204,7 +204,6 @@ Kylin ODBC é©±å¨å¯ä»¥ä»�
 
 
   http://kylin.apache.org/download/KylinODBCDriver.zip;>Kylin 
ODBC é©±å¨
-  http://kylin.apache.org/download/KylinODBCDriver.zip;>Kylin 
ODBC é©±å¨
 
 
 

Modified: kylin/site/community/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/community/index.html?rev=1721410=1721409=1721410=diff
==
--- kylin/site/community/index.html (original)
+++ kylin/site/community/index.html Tue Dec 22 14:20:24 2015
@@ -213,7 +213,7 @@
 For convenience, thereâs a forum style mailing list archives which not 
part of offical Apache archives:
 
 
-  http://apache-kylin-incubating.74782.x6.nabble.com;>Developer 
List archive on Nabble
+  http://apache-kylin.74782.x6.nabble.com;>Developer List archive 
on Nabble
 
 
 Social Media

Modified: kylin/site/development/howto_package.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_package.html?rev=1721410=1721409=1721410=diff
==
--- kylin/site/development/howto_package.html (original)
+++ kylin/site/development/howto_package.html Tue Dec 22 14:20:24 2015
@@ -447,6 +447,11 @@
 
 In order to generate binary package, maven and 
npm are pre-requisites.
 
+If youâre behind a proxy server, both npm and bower need be told with the 
proxy info before running ./script/package.sh:
+
+export http_proxy=http://your-proxy-host:port
+npm config set proxy 
http://your-proxy-host:port
+




Modified: kylin/site/development/howto_release.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_release.html?rev=1721410=1721409=1721410=diff
==
--- kylin/site/development/howto_release.html (original)
+++ kylin/site/development/howto_release.html Tue Dec 22 14:20:24 2015
@@ -591,28 +591,28 @@ If youâre the first time to do rele
 $ mvn clean
 
 # Do a dry run of the release:prepare step, which sets version 
numbers.
-$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z-incubating -DdevelopmentVersion=(X.Y.Z+1)-incubating-SNAPSHOT 
-Papache-release -Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE}" release:prepare 21 | tee 
/tmp/prepare-dry.log
+$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z -DdevelopmentVersion=(X.Y.Z+1)-SNAPSHOT -Papache-release 
-Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE}" release:prepare 21 | tee 
/tmp/prepare-dry.log
 
 Check the artifacts:
 
 
   In the target directory should be 
these 8 files, among others:
 
-  apache-kylin-X.Y.Z-incubating-SNAPSHOT-src.tar.gz
-  apache-kylin-X.Y.Z-incubating-SNAPSHOT-src.tar.gz.asc
-  apache-kylin-X.Y.Z-incubating-SNAPSHOT-src.tar.gz.md5

kylin git commit: KYLIN-1234 enable ACL check on query

2015-12-23 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 2055efcff -> 6826d6dd6


KYLIN-1234 enable ACL check on query

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6826d6dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6826d6dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6826d6dd

Branch: refs/heads/2.x-staging
Commit: 6826d6dd691fd6266aac9eaa0a9ce61e27a0dbf5
Parents: 2055efc
Author: shaofengshi <shaofeng...@apache.org>
Authored: Thu Dec 24 11:14:41 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Thu Dec 24 11:14:41 2015 +0800

--
 build/conf/kylin.properties| 2 ++
 .../src/main/java/org/apache/kylin/common/AbstractKylinConfig.java | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6826d6dd/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 41e127e..0796173 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -65,6 +65,8 @@ kylin.hbase.region.cut.small=5
 kylin.hbase.region.cut.medium=10
 kylin.hbase.region.cut.large=50
 
+# Enable/disable ACL check for cube query
+kylin.query.security.enabled=true
 
 ## kylin security configurations
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6826d6dd/core-common/src/main/java/org/apache/kylin/common/AbstractKylinConfig.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/AbstractKylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/AbstractKylinConfig.java
index a16346c..544bd1e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/AbstractKylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/AbstractKylinConfig.java
@@ -413,7 +413,7 @@ public class AbstractKylinConfig implements Serializable {
 }
 
 public boolean isQuerySecureEnabled() {
-return 
Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
+return 
Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "true"));
 }
 
 public boolean isQueryCacheEnabled() {

[1/2] kylin git commit: KYLIN-1250 fix NPE when discarding a job

2015-12-23 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 20104d3d9 -> 7a41f44c5


KYLIN-1250 fix NPE when discarding a job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87ac53b5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87ac53b5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87ac53b5

Branch: refs/heads/1.x-staging
Commit: 87ac53b5cd85e05ef0e58e8ebe60f75839acf686
Parents: 20104d3
Author: shaofengshi <shaofeng...@apache.org>
Authored: Thu Dec 24 10:58:04 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Thu Dec 24 11:15:23 2015 +0800

--
 .../main/java/org/apache/kylin/cube/CubeDescManager.java  | 10 +-
 .../org/apache/kylin/rest/controller/CubeController.java  |  8 
 .../java/org/apache/kylin/rest/service/JobService.java|  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/87ac53b5/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index dfb1b88..89f8b01 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -252,15 +252,7 @@ public class CubeDescManager {
 }
 
 desc.setSignature(desc.calculateSignature());
-
-// drop cube segments if signature changes
-CubeInstance cube = getCubeManager().getCube(desc.getName());
-if (cube != null && !StringUtils.equals(desc.getSignature(), 
cube.getDescriptor().getSignature())) {
-logger.info("Detect signature change of [" + desc.getName() + "], 
drop all existing segments");
-cube.getSegments().clear();
-getCubeManager().updateCube(cube);
-}
-
+
 // Save Source
 String path = desc.getResourcePath();
 getStore().putResource(path, desc, CUBE_DESC_SERIALIZER);

http://git-wip-us.apache.org/repos/asf/kylin/blob/87ac53b5/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 679e4e6..85cbae7 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -459,9 +459,17 @@ public class CubeController extends BasicController {
 }
 
 CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+String previousCubeDescSignature = 
cube.getDescriptor().getSignature();
 String projectName = (null == cubeRequest.getProject()) ? 
ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
 desc = cubeService.updateCubeAndDesc(cube, desc, projectName);
 
+// drop cube segments if signature changes
+if (!StringUtils.equals(desc.getSignature(), 
previousCubeDescSignature)) {
+logger.info("Detect signature change of [" + desc.getName() + 
"], drop all existing segments");
+cube = cubeService.getCubeManager().getCube(cubeName);
+cubeService.purgeCube(cube);
+}
+
 ProjectManager projectManager = cubeService.getProjectManager();
 if (!cubeService.isCubeInProject(projectName, cube)) {
 String owner = 
SecurityContextHolder.getContext().getAuthentication().getName();

http://git-wip-us.apache.org/repos/asf/kylin/blob/87ac53b5/server/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index 0903f83..1be9bd5 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -312,7 +312,7 @@ public class JobService extends BasicService {
 CubeInstance cubeInstance = 
getCubeManager().getCube(job.getRelatedCube());
 for (String segmentId : segmentIds) {
 final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
-if (segment.getStatus() == SegmentStatusEnum.NEW) {
+if (segment != null && segment.getStatus() == 
SegmentStatu

[2/2] kylin git commit: KYLIN-1234 enable ACL check on query

2015-12-23 Thread shaofengshi

KYLIN-1234 enable ACL check on query


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a41f44c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a41f44c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a41f44c

Branch: refs/heads/1.x-staging
Commit: 7a41f44c5a254189b938668d505c1c611818f253
Parents: 87ac53b
Author: shaofengshi <shaofeng...@apache.org>
Authored: Thu Dec 24 11:14:21 2015 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Thu Dec 24 11:15:24 2015 +0800

--
 common/src/main/java/org/apache/kylin/common/KylinConfig.java | 2 +-
 conf/kylin.properties | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7a41f44c/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 8a123dc..b043b2a 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -485,7 +485,7 @@ public class KylinConfig {
 }
 
 public boolean isQuerySecureEnabled() {
-return 
Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
+return 
Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "true"));
 }
 
 public boolean isQueryCacheEnabled() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/7a41f44c/conf/kylin.properties
--
diff --git a/conf/kylin.properties b/conf/kylin.properties
index 7f2af8a..68db224 100644
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -80,6 +80,9 @@ kylin.hbase.region.cut.large=100
 kylin.hbase.region.count.min=1
 kylin.hbase.region.count.max=500
 
+# Enable/disable ACL check for cube query
+kylin.query.security.enabled=true
+
 ## Config for Restful APP ##
 # database connection settings:
 ldap.server=

kylin git commit: Cleanup old storage after full regression test

2016-01-12 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 58c518188 -> ac045e155


Cleanup old storage after full regression test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac045e15
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac045e15
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac045e15

Branch: refs/heads/2.x-staging
Commit: ac045e155ef282e121f0d3112daab16bd250cc0e
Parents: 58c5181
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jan 13 10:09:05 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jan 13 10:34:01 2016 +0800

--
 .../org/apache/kylin/job/BuildCubeWithEngineTest.java | 11 ++-
 .../org/apache/kylin/job/BuildIIWithStreamTest.java   | 14 ++
 2 files changed, 24 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac045e15/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 53db99a..46bf1bb 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -33,6 +33,7 @@ import java.util.concurrent.Future;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractKylinTestCase;
 import org.apache.kylin.common.util.ClassUtil;
@@ -49,6 +50,7 @@ import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
 import org.apache.kylin.job.manager.ExecutableManager;
 import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
 import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
 import org.junit.After;
 import org.junit.Before;
@@ -103,9 +105,9 @@ public class BuildCubeWithEngineTest {
 
 @Before
 public void before() throws Exception {
-
 
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
 
+cleanupOldStorage();
 DeployUtil.initCliWorkDir();
 DeployUtil.deployMetadata();
 DeployUtil.overrideJobJarLocations();
@@ -322,4 +324,11 @@ public class BuildCubeWithEngineTest {
 return job.getId();
 }
 
+private int cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+
+int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+return exitCode;
+}
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac045e15/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index d4245cd..b420463 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -71,6 +71,8 @@ import 
org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.source.hive.HiveTableReader;
 import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -114,6 +116,11 @@ public class BuildIIWithStreamTest {
 }
 }
 }
+
+@AfterClass
+public void cleanup() throws Exception {
+cleanupOldStorage();
+}
 
 private String createIntermediateTable(IIDesc desc, KylinConfig 
kylinConfig) throws IOException {
 IIJoinedFlatTableDesc intermediateTableDesc = new 
IIJoinedFlatTableDesc(desc);
@@ -273,4 +280,11 @@ public class BuildIIWithStreamTest {
 return unsorted;
 }
 
+private int cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+
+int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+return exitCode;
+}
+
 }

kylin git commit: fix a compile error

2016-01-12 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging ac045e155 -> 05b7ca560


fix a compile error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/05b7ca56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/05b7ca56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/05b7ca56

Branch: refs/heads/2.x-staging
Commit: 05b7ca560c256deecbd95de52832bc3ad8691804
Parents: ac045e1
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jan 13 12:54:48 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jan 13 12:57:24 2016 +0800

--
 .../test/java/org/apache/kylin/job/BuildIIWithStreamTest.java| 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/05b7ca56/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index b420463..2317e62 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -118,7 +118,7 @@ public class BuildIIWithStreamTest {
 }
 
 @AfterClass
-public void cleanup() throws Exception {
+public static void cleanup() throws Exception {
 cleanupOldStorage();
 }
 
@@ -280,7 +280,7 @@ public class BuildIIWithStreamTest {
 return unsorted;
 }
 
-private int cleanupOldStorage() throws Exception {
+private static int cleanupOldStorage() throws Exception {
 String[] args = { "--delete", "true" };
 
 int exitCode = ToolRunner.run(new StorageCleanupJob(), args);

kylin git commit: KYLIN-1306 Null strings are not applied during fast cubing

2016-01-12 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5db430706 -> 829f5d312


KYLIN-1306 Null strings are not applied during fast cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/829f5d31
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/829f5d31
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/829f5d31

Branch: refs/heads/2.x-staging
Commit: 829f5d3129d0521d22d42bd43ba07a83bc2945be
Parents: 5db4307
Author: shaofengshi <shaofeng...@apache.org>
Authored: Tue Jan 12 16:23:04 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jan 12 17:28:34 2016 +0800

--
 .../InMemCubeBuilderInputConverter.java | 30 
 1 file changed, 30 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/829f5d31/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
index 3e9a67d..a1eb6da 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -17,10 +17,13 @@
 */
 package org.apache.kylin.cube.inmemcubing;
 
+import org.apache.kylin.common.util.Bytes;
+
 import java.util.List;
 import java.util.Map;
 
 import org.apache.kylin.common.util.Dictionary;
+import com.google.common.collect.Lists;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
 import org.apache.kylin.gridtable.GTInfo;
@@ -35,12 +38,15 @@ import org.apache.kylin.metadata.model.TblColRef;
  */
 public class InMemCubeBuilderInputConverter {
 
+public static final byte[] HIVE_NULL = Bytes.toBytes("\\N");
+
 private final CubeJoinedFlatTableDesc intermediateTableDesc;
 private final MeasureDesc[] measureDescs;
 private final MeasureIngester[] measureIngesters;
 private final int measureCount;
 private final Map<TblColRef, Dictionary> dictionaryMap;
 private final GTInfo gtInfo;
+protected List<byte[]> nullBytes;
 
 
 public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map<TblColRef, 
Dictionary> dictionaryMap, GTInfo gtInfo) {
@@ -50,6 +56,7 @@ public class InMemCubeBuilderInputConverter {
 this.measureDescs = cubeDesc.getMeasures().toArray(new 
MeasureDesc[measureCount]);
 this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures());
 this.dictionaryMap = dictionaryMap;
+initNullBytes(cubeDesc);
 }
 
 public final GTRecord convert(List row) {
@@ -73,6 +80,9 @@ public class InMemCubeBuilderInputConverter {
 
 for (int i = 0; i < keySize; i++) {
 key[i] = 
row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]);
+if (key[i] != null && isNull(Bytes.toBytes((String)key[i]))) {
+key[i] = null;
+}
 }
 
 return key;
@@ -112,4 +122,24 @@ public class InMemCubeBuilderInputConverter {
 return measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, 
dictionaryMap);
 }
 
+private void initNullBytes(CubeDesc cubeDesc) {
+nullBytes = Lists.newArrayList();
+nullBytes.add(HIVE_NULL);
+String[] nullStrings = cubeDesc.getNullStrings();
+if (nullStrings != null) {
+for (String s : nullStrings) {
+nullBytes.add(Bytes.toBytes(s));
+}
+}
+}
+
+private boolean isNull(byte[] v) {
+for (byte[] nullByte : nullBytes) {
+if (Bytes.equals(v, nullByte))
+return true;
+}
+return false;
+}
+
+
 }

kylin git commit: Cleanup old storage after full regression test

2016-01-12 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 7bfbbe2ff -> e0ff2a5c4


Cleanup old storage after full regression test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e0ff2a5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e0ff2a5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e0ff2a5c

Branch: refs/heads/2.0-rc
Commit: e0ff2a5c428aa046d8dbe37090914e2d07077da3
Parents: 7bfbbe2
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jan 13 10:09:05 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jan 13 10:17:47 2016 +0800

--
 .../org/apache/kylin/job/BuildCubeWithEngineTest.java | 11 ++-
 .../org/apache/kylin/job/BuildIIWithStreamTest.java   | 14 ++
 2 files changed, 24 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e0ff2a5c/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 53db99a..46bf1bb 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -33,6 +33,7 @@ import java.util.concurrent.Future;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractKylinTestCase;
 import org.apache.kylin.common.util.ClassUtil;
@@ -49,6 +50,7 @@ import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
 import org.apache.kylin.job.manager.ExecutableManager;
 import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
 import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
 import org.junit.After;
 import org.junit.Before;
@@ -103,9 +105,9 @@ public class BuildCubeWithEngineTest {
 
 @Before
 public void before() throws Exception {
-
 
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
 
+cleanupOldStorage();
 DeployUtil.initCliWorkDir();
 DeployUtil.deployMetadata();
 DeployUtil.overrideJobJarLocations();
@@ -322,4 +324,11 @@ public class BuildCubeWithEngineTest {
 return job.getId();
 }
 
+private int cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+
+int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+return exitCode;
+}
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/e0ff2a5c/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index d4245cd..b420463 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -71,6 +71,8 @@ import 
org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.source.hive.HiveTableReader;
 import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -114,6 +116,11 @@ public class BuildIIWithStreamTest {
 }
 }
 }
+
+@AfterClass
+public void cleanup() throws Exception {
+cleanupOldStorage();
+}
 
 private String createIntermediateTable(IIDesc desc, KylinConfig 
kylinConfig) throws IOException {
 IIJoinedFlatTableDesc intermediateTableDesc = new 
IIJoinedFlatTableDesc(desc);
@@ -273,4 +280,11 @@ public class BuildIIWithStreamTest {
 return unsorted;
 }
 
+private int cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+
+int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+return exitCode;
+}
+
 }

[1/2] kylin git commit: update howto_release doc

2016-06-08 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document ac0942344 -> 5834ded3d


update howto_release doc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1cfb28b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1cfb28b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1cfb28b8

Branch: refs/heads/document
Commit: 1cfb28b85451c2697dbbab7c16478d9cc36b22e4
Parents: ac09423
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jun 8 16:14:20 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jun 8 16:14:40 2016 +0800

--
 website/_dev/howto_package.md |  2 +-
 website/_dev/howto_release.md | 14 +-
 2 files changed, 14 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1cfb28b8/website/_dev/howto_package.md
--
diff --git a/website/_dev/howto_package.md b/website/_dev/howto_package.md
index a9dec33..f0f044f 100644
--- a/website/_dev/howto_package.md
+++ b/website/_dev/howto_package.md
@@ -9,7 +9,7 @@ permalink: /development/howto_package.html
 {% highlight bash %}
 git clone https://github.com/apache/kylin kylin
 cd kylin
-./script/package.sh
+./build/script/package.sh
 {% endhighlight %}
 
 In order to generate binary package, **maven** and **npm** are pre-requisites.

http://git-wip-us.apache.org/repos/asf/kylin/blob/1cfb28b8/website/_dev/howto_release.md
--
diff --git a/website/_dev/howto_release.md b/website/_dev/howto_release.md
index d2b4ab9..e84cc02 100644
--- a/website/_dev/howto_release.md
+++ b/website/_dev/howto_release.md
@@ -18,7 +18,7 @@ Make sure you have avaliable account and privilege for 
following applications:
 * Apache Nexus (maven repo): 
[https://repository.apache.org](https://repository.apache.org)  
 * Apache Kylin dist repo: 
[https://dist.apache.org/repos/dist/dev/kylin](https://dist.apache.org/repos/dist/dev/kylin)
  
 
-## Setup PGP signing keys  
+## Setup GPG signing keys  
 Follow instructions at 
[http://www.apache.org/dev/release-signing](http://www.apache.org/dev/release-signing)
 to create a key pair  
 Install gpg (On Mac OS X as sample):  
 `brew install gpg`
@@ -63,6 +63,7 @@ __Before you start:__
 
 * Set up signing keys as described above.
 * Make sure you are using JDK 1.7 (not 1.8).
+* Make sure you are using GIT 2.7.2 or above.
 * Make sure you are working on right release version number.
 * Make sure that every âresolvedâ JIRA case (including duplicates) has a 
fix version assigned.
 
@@ -412,6 +413,17 @@ The old releases will remain available in the
 
 Release same version in JIRA, check [Change 
Log](https://issues.apache.org/jira/browse/KYLIN/?selectedTab=com.atlassian.jira.jira-projects-plugin:changelog-panel)
 for the latest released version.
 
+## Build and upload binary package
+After publish the release, you need generate the binary packages and then put 
them to the svn release repository;
+
+* Git checkout the tag for current release; 
+* Make a binary package by refering to [this doc](howto_package.html);
+* Sign the generated binary package with gpg, e.g,: "gpg --armor --output 
apache-kylin-1.5.0-bin.tar.gz.asc --detach-sig apache-kylin-1.5.0-bin.tar.gz"
+* Move both the binary package and the signature file to the svn release 
folder for this version;
+* For different Hadoop/HBase version, you may need repeat the above steps;
+* Add the files and then commit the svn changes. 
+
+
 ## Update source code
 After publish the release, you need to manually update some source code:

[2/2] kylin git commit: update announcement letter

2016-06-08 Thread shaofengshi

update announcement letter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5834ded3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5834ded3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5834ded3

Branch: refs/heads/document
Commit: 5834ded3d165b1ba954c9a9232d8f14439e7e8b0
Parents: 1cfb28b
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jun 8 23:21:15 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jun 8 23:21:15 2016 +0800

--
 website/_dev/howto_release.md | 30 +-
 1 file changed, 29 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5834ded3/website/_dev/howto_release.md
--
diff --git a/website/_dev/howto_release.md b/website/_dev/howto_release.md
index e84cc02..2365f40 100644
--- a/website/_dev/howto_release.md
+++ b/website/_dev/howto_release.md
@@ -438,7 +438,35 @@ Send one mail with subject like "[Announce] Apache Kylin 
x.y released" to follow
 * Apache Kylin Dev mailing list: d...@kylin.apache.org
 * Apache Kylin User mailing list: u...@kylin.apache.org
 * Apache Announce mailing list: annou...@apache.org
-Please notice to always use your Apache mail address to send this
+Please notice to always use your Apache mail address to send this;
+
+Here is a sample of announcement email (by studying Kafka's):
+
+{% highlight text %} 
+The Apache Kylin team is pleased to announce the immediate availability of the 
1.5.2.1 release. 
+
+This is a bug fix release based on 1.5.2; All of the changes in this release 
can be found:
+https://kylin.apache.org/docs15/release_notes.html
+
+You can download the source release and binary packages from 
+https://www.apache.org/dyn/closer.cgi?path=/kylin/apache-kylin-1.5.2.1/
+
+More information about the binary packages is on Kylin's download page 
https://kylin.apache.org/download/
+
+Apache Kylin is an open source Distributed Analytics Engine designed to 
provide SQL interface and multi-dimensional analysis (OLAP) on Hadoop, 
supporting extremely large datasets.
+
+Apache Kylin lets you query big Hive tables at sub-second latency in 3 simple 
steps:
+1. Identify a set of Hive tables in star schema.
+2. Build a cube from the Hive tables in an offline batch process.
+3. Query the Hive tables using SQL and get results in sub-seconds, via Rest 
API, ODBC, or JDBC.
+
+Thanks everyone who have contributed to the 1.5.2.1 release.
+
+We welcome your help and feedback. For more information on how to
+report problems, and to get involved, visit the project website at
+https://kylin.apache.org/
+
+{% endhighlight %}
 
 # Thanks  
 This guide drafted with reference from [Apache 
Calcite](http://calcite.apache.org) Howto doc, Thank you very much.

kylin git commit: KYLIN-1752 Add an option to fail cube build job when source table is empty

2016-06-07 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 9f1029a76 -> d2e96bda0


KYLIN-1752 Add an option to fail cube build job when source table is empty

Signed-off-by: shaofengshi <shaofeng...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d2e96bda
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d2e96bda
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d2e96bda

Branch: refs/heads/master
Commit: d2e96bda0a5848815cf4430746276bc9e896bad7
Parents: 9f1029a
Author: gaodayue <gaoda...@meituan.com>
Authored: Fri Jun 3 18:17:43 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jun 7 15:09:29 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 
 .../source/hive/CreateFlatHiveTableStep.java| 22 +---
 2 files changed, 18 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d2e96bda/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 68e3b6c..7664c66 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -363,6 +363,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.job.cmd.extra.args");
 }
 
+public boolean isEmptySegmentAllowed() {
+return 
Boolean.parseBoolean(getOptional("kylin.job.allow.empty.segment", "true"));
+}
+
 public String getOverrideHiveTableLocation(String table) {
 return getOptional("hive.table.location." + table.toUpperCase());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d2e96bda/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index e9b9994..443de99 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -23,9 +23,11 @@ import java.io.InputStream;
 public class CreateFlatHiveTableStep extends AbstractExecutable {
 private final BufferedLogger stepLogger = new BufferedLogger(logger);
 
-private long readRowCountFromFile(Path file) throws IOException {
-FileSystem fs = FileSystem.get(file.toUri(), 
HadoopUtil.getCurrentConfiguration());
-InputStream in = fs.open(file);
+private long readRowCountFromFile() throws IOException {
+Path rowCountFile = new Path(getRowCountOutputDir(), "00_0");
+
+FileSystem fs = FileSystem.get(rowCountFile.toUri(), 
HadoopUtil.getCurrentConfiguration());
+InputStream in = fs.open(rowCountFile);
 try {
 String content = IOUtils.toString(in);
 return Long.valueOf(content.trim()); // strip the '\n' character
@@ -35,9 +37,7 @@ public class CreateFlatHiveTableStep extends 
AbstractExecutable {
 }
 }
 
-private int determineNumReducer(KylinConfig config) throws IOException {
-Path rowCountFile = new Path(getRowCountOutputDir(), "00_0");
-long rowCount = readRowCountFromFile(rowCountFile);
+private int determineNumReducer(KylinConfig config, long rowCount) throws 
IOException {
 int mapperInputRows = config.getHadoopJobMapperInputRows();
 
 int numReducers = Math.round(rowCount / ((float) mapperInputRows));
@@ -78,8 +78,14 @@ public class CreateFlatHiveTableStep extends 
AbstractExecutable {
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
 KylinConfig config = getCubeSpecificConfig();
 try {
-
-int numReducers = determineNumReducer(config);
+long rowCount = readRowCountFromFile();
+if (!config.isEmptySegmentAllowed() && rowCount == 0) {
+stepLogger.log("Detect upstream hive table is empty, " +
+"fail the job because 
\"kylin.job.allow.empty.segment\" = \"false\"");
+return new ExecuteResult(ExecuteResult.State.ERROR, 
stepLogger.getBufferedLog());
+}
+
+int numReducers = determineNumReducer(config, rowCount);
 createFlatHiveTable(config, numReducers);
 return new ExecuteResult(ExecuteResult.State.SUCCEED, 
stepLogger.getBufferedLog());

[kylin] Git Push Summary

2016-06-07 Thread shaofengshi

Repository: kylin
Updated Tags:  refs/tags/kylin-1.5.2.1-HBase1.x [created] 2b2b3a228

[4/8] kylin git commit: KYLIN-1758 add the hive set statements

2016-06-07 Thread shaofengshi

KYLIN-1758 add the hive set statements


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1d17e21
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1d17e21
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1d17e21

Branch: refs/heads/1.5.x-HBase1.x
Commit: a1d17e215561089bb5958b84e876a74fc60f092f
Parents: ef1d13f
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sat Jun 4 16:22:38 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sat Jun 4 16:22:38 2016 +0800

--
 .../org/apache/kylin/job/JoinedFlatTable.java   | 56 +++-
 .../apache/kylin/source/hive/HiveMRInput.java   | 14 +
 2 files changed, 44 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a1d17e21/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 5886325..f13a48c 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -55,6 +55,36 @@ public class JoinedFlatTable {
 return storageDfsDir + "/" + intermediateTableDesc.getTableName();
 }
 
+public static String generateHiveSetStatements(JobEngineConfig 
engineConfig) throws IOException {
+StringBuilder buffer = new StringBuilder();
+File hadoopPropertiesFile = new 
File(engineConfig.getHiveConfFilePath());
+
+if (hadoopPropertiesFile.exists()) {
+DocumentBuilderFactory factory = 
DocumentBuilderFactory.newInstance();
+DocumentBuilder builder;
+Document doc;
+try {
+builder = factory.newDocumentBuilder();
+doc = builder.parse(hadoopPropertiesFile);
+NodeList nl = doc.getElementsByTagName("property");
+for (int i = 0; i < nl.getLength(); i++) {
+String name = 
doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
+String value = 
doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
+if (!name.equals("tmpjars")) {
+buffer.append("SET " + name + "=" + value + ";\n");
+}
+}
+
+} catch (ParserConfigurationException e) {
+throw new IOException(e);
+} catch (SAXException e) {
+throw new IOException(e);
+}
+}
+
+return buffer.toString();
+}
+
 public static String generateCreateTableStatement(IJoinedFlatTableDesc 
intermediateTableDesc, String storageDfsDir) {
 StringBuilder ddl = new StringBuilder();
 
@@ -86,32 +116,6 @@ public class JoinedFlatTable {
 
 public static String generateInsertDataStatement(IJoinedFlatTableDesc 
intermediateTableDesc, JobEngineConfig engineConfig) throws IOException {
 StringBuilder sql = new StringBuilder();
-
-File hadoopPropertiesFile = new 
File(engineConfig.getHiveConfFilePath());
-
-if (hadoopPropertiesFile.exists()) {
-DocumentBuilderFactory factory = 
DocumentBuilderFactory.newInstance();
-DocumentBuilder builder;
-Document doc;
-try {
-builder = factory.newDocumentBuilder();
-doc = builder.parse(hadoopPropertiesFile);
-NodeList nl = doc.getElementsByTagName("property");
-for (int i = 0; i < nl.getLength(); i++) {
-String name = 
doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
-String value = 
doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
-if (name.equals("tmpjars") == false) {
-sql.append("SET " + name + "=" + value + 
";").append("\n");
-}
-}
-
-} catch (ParserConfigurationException e) {
-throw new IOException(e);
-} catch (SAXException e) {
-throw new IOException(e);
-}
-}
-
 sql.append("INSERT OVERWRITE TABLE " + 
intermediateTableDesc.getTableName() + " " + 
generateSelectDataStatement(intermediateTableDesc) + ";").append("\n");
 
 return sql.toString();

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1d17e21/source-hive/s

[3/8] kylin git commit: update version number

2016-06-07 Thread shaofengshi

update version number


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef1d13f8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef1d13f8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef1d13f8

Branch: refs/heads/1.5.x-HBase1.x
Commit: ef1d13f8482b755b59c077c5b1ae1898e2e05e66
Parents: 6688d64
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sat Jun 4 08:24:38 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sat Jun 4 08:24:38 2016 +0800

--
 assembly/pom.xml | 2 +-
 atopcalcite/pom.xml  | 2 +-
 core-common/pom.xml  | 2 +-
 core-cube/pom.xml| 2 +-
 core-dictionary/pom.xml  | 2 +-
 core-job/pom.xml | 2 +-
 core-metadata/pom.xml| 2 +-
 core-storage/pom.xml | 2 +-
 engine-mr/pom.xml| 2 +-
 engine-spark/pom.xml | 2 +-
 engine-streaming/pom.xml | 2 +-
 invertedindex/pom.xml| 2 +-
 jdbc/pom.xml | 2 +-
 kylin-it/pom.xml | 2 +-
 pom.xml  | 4 ++--
 query/pom.xml| 2 +-
 server/pom.xml   | 2 +-
 source-hive/pom.xml  | 2 +-
 source-kafka/pom.xml | 2 +-
 storage-hbase/pom.xml| 2 +-
 tool/pom.xml | 2 +-
 21 files changed, 22 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index b6e3675..204035b 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.5.2
+1.5.2.1
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 7c70df3..7779201 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 7647621..5030d5a 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 2309872..f4e0c71 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 1af0091..a6cd19c 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 864bf1b..383303c 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index f65129d..184dc5a 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 2f08dd9..4948e59 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/engine-mr/pom.xml
--
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 379eaa6..dcd3e2f 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-

[1/8] kylin git commit: KYLIN-1739 Do not cache and reuse hadoop configuration in HadoopUtil [Forced Update!]

2016-06-07 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.5.x-HBase1.x e85b49823 -> cf4d2940b (forced update)


KYLIN-1739 Do not cache and reuse hadoop configuration in HadoopUtil


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0860934f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0860934f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0860934f

Branch: refs/heads/1.5.x-HBase1.x
Commit: 0860934f8bf7e30256d2c59c862c63d6b77d87ed
Parents: af2646b
Author: Li Yang <liy...@apache.org>
Authored: Thu May 26 14:00:01 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Fri Jun 3 18:30:47 2016 +0800

--
 .../src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java  | 3 +--
 .../apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java  | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0860934f/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 9ce2bab..c638d99 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -41,8 +41,7 @@ public class HadoopUtil {
 public static Configuration getCurrentConfiguration() {
 if (hadoopConfig.get() == null) {
 Configuration conf = healSickConfig(new Configuration());
-
-hadoopConfig.set(conf);
+return conf;
 }
 return hadoopConfig.get();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0860934f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 12b1fa8..f04d46e 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -90,7 +90,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 this.expectedSize = expectedSize;
 this.queue = new ArrayBlockingQueue<byte[]>(expectedSize);
 
-this.timeout = 
HadoopUtil.getCurrentConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 
HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+this.timeout = 
HBaseConnection.getCurrentHBaseConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
 HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
 this.timeout *= 
KylinConfig.getInstanceFromEnv().getCubeVisitTimeoutTimes();
 
 if (BackdoorToggles.getQueryTimeout() != -1) {

[8/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-06-07 Thread shaofengshi

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cf4d2940
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cf4d2940
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cf4d2940

Branch: refs/heads/1.5.x-HBase1.x
Commit: cf4d2940b67d622eacd2ac9a913b221091a35c2e
Parents: 71a847b
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jun 7 16:29:01 2016 +0800

--
 .../kylin/job/hadoop/invertedindex/IITest.java  | 10 ++-
 .../apache/kylin/common/KylinConfigBase.java|  5 +-
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../test_case_data/sandbox/kylin_job_conf.xml   | 86 +-
 examples/test_case_data/sandbox/mapred-site.xml | 23 +++--
 .../kylin/provision/BuildCubeWithEngine.java| 54 +--
 .../kylin/provision/BuildIIWithStream.java  | 13 +--
 .../hbase/ii/ITInvertedIndexHBaseTest.java  |  9 +-
 pom.xml | 18 ++--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 38 +++-
 .../apache/kylin/rest/service/QueryService.java | 21 +++--
 .../apache/kylin/rest/service/UserService.java  | 16 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 36 
 .../kylin/storage/hbase/HBaseResourceStore.java | 29 +++---
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/HBaseClientKVIterator.java| 11 +--
 .../hbase/cube/v1/RegionScannerAdapter.java | 11 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 15 +++-
 .../observer/ObserverAggregationCache.java  | 15 ++--
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 11 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/ii/IICreateHTableJob.java | 13 +--
 .../hbase/ii/InvertedIndexStorageQuery.java |  6 +-
 .../endpoint/EndpointTupleIterator.java | 13 +--
 .../ii/coprocessor/endpoint/IIEndpoint.java |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 24 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 13 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 38 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 23 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 42 -
 .../kylin/storage/hbase/util/HBaseUsage.java| 10 ++-
 .../storage/hbase/util/HbaseStreamingInput.java | 31 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 18 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 16 ++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 31 +++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 59 files changed, 530 insertions(+), 583 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cf4d2940/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java 
b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index da25143..5d2cfc4 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -38,6 +38,7 @@ import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoop.hbase.C

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-06-07 Thread shaofengshi

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/add37856
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/add37856
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/add37856

Branch: refs/heads/kylin-1.5.2.1-cdh5.7-branch
Commit: add37856b0e9701c1ff55351c09f02525582a99d
Parents: 71a847b
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jun 7 16:45:29 2016 +0800

--
 .../kylin/job/hadoop/invertedindex/IITest.java  | 10 ++-
 .../apache/kylin/common/KylinConfigBase.java|  5 +-
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../test_case_data/sandbox/kylin_job_conf.xml   | 86 +-
 examples/test_case_data/sandbox/mapred-site.xml | 23 +++--
 .../kylin/provision/BuildCubeWithEngine.java| 54 +--
 .../kylin/provision/BuildIIWithStream.java  | 13 +--
 .../hbase/ii/ITInvertedIndexHBaseTest.java  |  9 +-
 pom.xml | 18 ++--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 38 +++-
 .../apache/kylin/rest/service/QueryService.java | 21 +++--
 .../apache/kylin/rest/service/UserService.java  | 16 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 36 
 .../kylin/storage/hbase/HBaseResourceStore.java | 29 +++---
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/HBaseClientKVIterator.java| 11 +--
 .../hbase/cube/v1/RegionScannerAdapter.java | 11 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 15 +++-
 .../observer/ObserverAggregationCache.java  | 15 ++--
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 11 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/ii/IICreateHTableJob.java | 13 +--
 .../hbase/ii/InvertedIndexStorageQuery.java |  6 +-
 .../endpoint/EndpointTupleIterator.java | 13 +--
 .../ii/coprocessor/endpoint/IIEndpoint.java |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 24 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 13 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 38 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 23 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 42 -
 .../kylin/storage/hbase/util/HBaseUsage.java| 10 ++-
 .../storage/hbase/util/HbaseStreamingInput.java | 31 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 18 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 16 ++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 31 +++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 59 files changed, 530 insertions(+), 583 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/add37856/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java 
b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index da25143..5d2cfc4 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -38,6 +38,7 @@ import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoo

[1/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-06-07 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/kylin-1.5.2.1-cdh5.7-branch [created] e760242da


KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang <liy...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e760242d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e760242d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e760242d

Branch: refs/heads/kylin-1.5.2.1-cdh5.7-branch
Commit: e760242daad739b66813dc2c2d8a52bf74c0ec22
Parents: add3785
Author: Lynne Jiang <lynne...@hotmail.com>
Authored: Mon May 16 03:33:27 2016 -0700
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jun 7 16:45:29 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 396 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  18 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 10 files changed, 418 insertions(+), 1129 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e760242d/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/e760242d/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e760242d/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op

[kylin] Git Push Summary

2016-06-07 Thread shaofengshi

Repository: kylin
Updated Tags:  refs/tags/kylin-1.5.2.1-cdh5.7 [created] e760242da
  refs/tags/kylin-2.0-alpha [created] e159079f4
  refs/tags/v1.5.2-release [created] 354a13a36

[6/8] kylin git commit: [maven-release-plugin] prepare release kylin-1.5.2.1

2016-06-07 Thread shaofengshi

[maven-release-plugin] prepare release kylin-1.5.2.1


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71a847bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71a847bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71a847bb

Branch: refs/heads/1.5.x-HBase1.x
Commit: 71a847bbe97ec4e6f23b1bf271132d6ff9955a96
Parents: 404c530
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sat Jun 4 11:51:30 2016 +
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sat Jun 4 11:51:30 2016 +

--
 assembly/pom.xml | 2 +-
 atopcalcite/pom.xml  | 2 +-
 core-common/pom.xml  | 2 +-
 core-cube/pom.xml| 2 +-
 core-dictionary/pom.xml  | 2 +-
 core-job/pom.xml | 2 +-
 core-metadata/pom.xml| 2 +-
 core-storage/pom.xml | 2 +-
 engine-mr/pom.xml| 2 +-
 engine-spark/pom.xml | 2 +-
 engine-streaming/pom.xml | 2 +-
 invertedindex/pom.xml| 2 +-
 jdbc/pom.xml | 2 +-
 kylin-it/pom.xml | 2 +-
 pom.xml  | 2 +-
 query/pom.xml| 2 +-
 server/pom.xml   | 2 +-
 source-hive/pom.xml  | 2 +-
 source-kafka/pom.xml | 2 +-
 storage-hbase/pom.xml| 2 +-
 tool/pom.xml | 2 +-
 21 files changed, 21 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/71a847bb/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 2aa8c87..204035b 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/71a847bb/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index ec23344..7779201 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/71a847bb/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 381192a..5030d5a 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/71a847bb/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0fa5ffe..f4e0c71 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/71a847bb/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 56d70e7..a6cd19c 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/71a847bb/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 8a0ee7d..383303c 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/71a847bb/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 76d709c..184dc5a 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/71a847bb/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index b9daa31..4948e59 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/71a847bb/engine-mr/pom.xml
--
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 2d97310..dcd3e2f 100644
--- a/engin

[kylin] Git Push Summary

2016-06-07 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/kylin-1.5.2.1-cdh5.7-branch [deleted] e760242da

[kylin] Git Push Summary

2016-06-07 Thread shaofengshi

Repository: kylin
Updated Tags:  refs/tags/v1.5.2-release [deleted] 354a13a36

[kylin] Git Push Summary

2016-06-07 Thread shaofengshi

Repository: kylin
Updated Tags:  refs/tags/kylin-2.0-alpha [deleted] e159079f4

[1/3] kylin git commit: KYLIN-1755 TopN support constant as counter

2016-06-06 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 9ad34ec98 -> 4668bd31e


KYLIN-1755 TopN support constant as counter

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4668bd31
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4668bd31
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4668bd31

Branch: refs/heads/master
Commit: 4668bd31e010bec16d0a61f85cb4824ad42c237e
Parents: 9113a46
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Jun 6 18:21:16 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Jun 6 18:28:56 2016 +0800

--
 .../kylin/measure/topn/TopNMeasureType.java | 22 +---
 1 file changed, 15 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4668bd31/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 0ff5b3a..26519fd 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -254,18 +254,21 @@ public class TopNMeasureType extends 
MeasureType<TopNCounter> {
 if (sum == null)
 return false;
 
-if (!isTopN(topN) || !sum.isSum())
+if (!isTopN(topN))
 return false;
 
 TblColRef topnNumCol = getTopNNumericColumn(topN);
 
-if (sum.getParameter().getColRefs().isEmpty()) {
-if (topnNumCol == null && 
sum.getParameter().getValue().equals(topN.getParameter().getValue()))
+if (topnNumCol == null) {
+if (sum.isCount())
 return true;
-else
-return false;
+
+return false;
 }
 
+if (sum.isSum() == false)
+return false;
+
 TblColRef sumCol = sum.getParameter().getColRefs().get(0);
 return sumCol.equals(topnNumCol);
 }
@@ -294,7 +297,7 @@ public class TopNMeasureType extends 
MeasureType<TopNCounter> {
 
 if (sqlDigest.aggregations.size() > 0) {
 FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
-if (origFunc.isSum() == false) {
+if (origFunc.isSum() == false && origFunc.isCount() == false) {
 throw new IllegalStateException("When query with topN, only 
SUM function is allowed.");
 }
 logger.info("Rewrite function " + origFunc + " to " + topnFunc);
@@ -327,7 +330,12 @@ public class TopNMeasureType extends 
MeasureType<TopNCounter> {
 }
 
 // for TopN, the aggr must be SUM, so the number fill into the column 
position (without rewrite)
-final int numericTupleIdx = (numericCol != null && 
tupleInfo.hasColumn(numericCol)) ? tupleInfo.getColumnIndex(numericCol) : -1;
+final int numericTupleIdx;
+if (numericCol != null ) {
+numericTupleIdx = tupleInfo.hasColumn(numericCol) ? 
tupleInfo.getColumnIndex(numericCol) : -1;
+} else {
+numericTupleIdx = tupleInfo.getFieldIndex("COUNT__");
+}
 return new IAdvMeasureFiller() {
 private TopNCounter topNCounter;
 private Iterator<Counter> topNCounterIterator;

[2/3] kylin git commit: KYLIN-1478 support non-dic encoding

2016-06-06 Thread shaofengshi

KYLIN-1478 support non-dic encoding

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9113a467
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9113a467
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9113a467

Branch: refs/heads/master
Commit: 9113a4675430a0baffb8d133ba29465c5cd70b4a
Parents: a490224
Author: shaofengshi <shaofeng...@apache.org>
Authored: Mon Jun 6 17:37:11 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Jun 6 18:28:56 2016 +0800

--
 .../apache/kylin/cube/model/RowKeyColDesc.java  |  11 +-
 .../metadata/measure/TopNMeasureTypeTest.java   |  62 ++
 .../kylin/dimension/DimensionEncoding.java  |  13 ++
 .../apache/kylin/dimension/IntegerDimEnc.java   |  16 ++
 .../kylin/measure/topn/TopNMeasureType.java | 202 +--
 .../kylin/metadata/model/FunctionDesc.java  |  14 +-
 .../kylin/metadata/model/MeasureDesc.java   |   2 +
 ...t_kylin_cube_without_slr_left_join_desc.json |   3 +-
 .../test_kylin_inner_join_model_desc.json   |  17 +-
 .../test_kylin_left_join_model_desc.json|  16 +-
 .../test_case_data/sandbox/kylin_hive_conf.xml  |  12 +-
 11 files changed, 261 insertions(+), 107 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 7e03355..e1be041 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.cube.model;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -57,13 +58,9 @@ public class RowKeyColDesc {
 public void init() {
 Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
 
-String[] parts = this.encoding.split("\\s*[(),:]\\s*");
-if (parts == null || parts.length == 0 || parts[0].isEmpty())
-throw new IllegalArgumentException("Not supported row key col 
encoding: '" + this.encoding + "'");
-
-this.encodingName = parts[0];
-this.encodingArgs = parts[parts.length - 1].isEmpty() //
-? StringUtil.subArray(parts, 1, parts.length - 1) : 
StringUtil.subArray(parts, 1, parts.length);
+Object[] encodingConf = 
DimensionEncoding.parseEncodingConf(this.encoding);
+this.encodingName = (String) encodingConf[0];
+this.encodingArgs = (String[])encodingConf[1];
 
 if (!DimensionEncodingFactory.isVaildEncoding(this.encodingName))
 throw new IllegalArgumentException("Not supported row key col 
encoding: '" + this.encoding + "'");

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
new file mode 100644
index 000..3d3e7d0
--- /dev/null
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
@@ -0,0 +1,62 @@
+package org.apache.kylin.metadata.measure;
+
+import static org.junit.Assert.*;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.topn.TopNMeasureType;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * Created by shishaofeng on 6/6/16.
+ */
+public class TopNMeasureTypeTest extends LocalFileMetadataTestCase {
+
+
+@Before
+public void setup() {
+this.createTestMetadata();
+
+}
+
+@After
+public void clear() {
+this.cleanupTestMetadata();
+}
+
+@Test
+public void test() {
+
+CubeDesc desc = 
CubeDescManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_without_s

[3/3] kylin git commit: KYLIN-1478 initial change

2016-06-06 Thread shaofengshi

KYLIN-1478 initial change


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a490224e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a490224e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a490224e

Branch: refs/heads/master
Commit: a490224eea84fd400320f8e29732c7371a5d0a73
Parents: 9ad34ec
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jun 1 22:24:11 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Mon Jun 6 18:28:56 2016 +0800

--
 .../apache/kylin/measure/topn/TopNMeasureType.java |  6 +-
 .../apache/kylin/metadata/model/FunctionDesc.java  | 17 +
 2 files changed, 18 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a490224e/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index d17c7d5..009edef 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -52,6 +52,10 @@ public class TopNMeasureType extends 
MeasureType<TopNCounter> {
 public static final String FUNC_TOP_N = "TOP_N";
 public static final String DATATYPE_TOPN = "topn";
 
+public static final String CONFIG_ENCODING_PREFIX = "topn.encoding.";
+public static final String CONFIG_AGG = "topn.aggregation";
+public static final String CONFIG_ORDER = "topn.order";
+
 public static class Factory extends 
MeasureTypeFactory<TopNCounter> {
 
 @Override
@@ -95,7 +99,7 @@ public class TopNMeasureType extends 
MeasureType<TopNCounter> {
 if (DATATYPE_TOPN.equals(dataType.getName()) == false)
 throw new IllegalArgumentException();
 
-if (dataType.getPrecision() < 1 || dataType.getPrecision() > 5000)
+if (dataType.getPrecision() < 1 || dataType.getPrecision() > 1)
 throw new IllegalArgumentException();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/a490224e/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index f237e73..0906d5f 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -18,10 +18,7 @@
 
 package org.apache.kylin.metadata.model;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.MeasureTypeFactory;
@@ -64,6 +61,9 @@ public class FunctionDesc {
 @JsonProperty("returntype")
 private String returnType;
 
+@JsonProperty("configurations")
+private HashMap<String, String> configurations = new LinkedHashMap<String, 
String>();
+
 private DataType returnDataType;
 private MeasureType measureType;
 private boolean isDimensionAsMetric = false;
@@ -265,6 +265,15 @@ public class FunctionDesc {
 return null;
 }
 
+public HashMap<String, String> getConfigurations() {
+return configurations;
+}
+
+public void setConfigurations(HashMap<String, String> configurations) {
+this.configurations = configurations;
+}
+
+
 @Override
 public int hashCode() {
 final int prime = 31;

svn commit: r13935 - in /release/kylin: apache-kylin-1.0-incubating/ apache-kylin-1.1-incubating/ apache-kylin-1.1.1-incubating/ apache-kylin-1.2/ apache-kylin-1.5.0/ apache-kylin-1.5.1/ apache-kylin-

2016-06-08 Thread shaofengshi

Author: shaofengshi
Date: Wed Jun  8 07:43:31 2016
New Revision: 13935

Log:
delete old releases

Removed:
release/kylin/apache-kylin-1.0-incubating/
release/kylin/apache-kylin-1.1-incubating/
release/kylin/apache-kylin-1.1.1-incubating/
release/kylin/apache-kylin-1.2/
release/kylin/apache-kylin-1.5.0/
release/kylin/apache-kylin-1.5.1/
release/kylin/apache-kylin-1.5.2/

svn commit: r1747361 - in /kylin/site: blog/index.html development/howto_package.html development/howto_release.html feed.xml

2016-06-08 Thread shaofengshi

Author: shaofengshi
Date: Wed Jun  8 08:23:21 2016
New Revision: 1747361

URL: http://svn.apache.org/viewvc?rev=1747361=rev
Log:
update howto_release

Modified:
kylin/site/blog/index.html
kylin/site/development/howto_package.html
kylin/site/development/howto_release.html
kylin/site/feed.xml

Modified: kylin/site/blog/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/blog/index.html?rev=1747361=1747360=1747361=diff
==
--- kylin/site/blog/index.html (original)
+++ kylin/site/blog/index.html Wed Jun  8 08:23:21 2016
@@ -180,25 +180,25 @@
 
 
 
-  Apache Kylin v1.5.2 
æ£å¼åå¸posted: May 26, 
2016
+  Apache 
Kylin v1.5.2 Release Announcementposted: May 26, 2016
 
   
 
 
 
-  Apache 
Kylin v1.5.2 Release Announcementposted: May 26, 2016
+  Apache Kylin v1.5.2 
æ£å¼åå¸posted: May 26, 
2016
 
   
 
 
 
-  Apache Kylin v1.5.1 
æ£å¼åå¸posted: Apr 12, 
2016
+  Apache 
Kylin v1.5.1 Release Announcementposted: Apr 12, 2016
 
   
 
 
 
-  Apache 
Kylin v1.5.1 Release Announcementposted: Apr 12, 2016
+  Apache Kylin v1.5.1 
æ£å¼åå¸posted: Apr 12, 
2016
 
   
 
@@ -222,13 +222,13 @@
 
 
 
-  Apache Kylin v1.3.0 
æ£å¼åå¸posted: Mar 16, 
2016
+  Apache 
Kylin v1.3.0 Release Announcementposted: Mar 16, 2016
 
   
 
 
 
-  Apache 
Kylin v1.3.0 Release Announcementposted: Mar 16, 2016
+  Apache Kylin v1.3.0 
æ£å¼åå¸posted: Mar 16, 
2016
 
   
 
@@ -246,13 +246,13 @@
 
 
 
-  Apache Kylinå¢å 
å¯¹Tableau 9åå¾®è½¯Excel, Power BIçæ¯æposted: Dec 25, 2015
+  Apache Kylin supports Tableau 
9 and MS Excel, Power BI nowposted: Dec 25, 2015
 
   
 
 
 
-  Apache Kylin supports Tableau 
9 and MS Excel, Power BI nowposted: Dec 25, 2015
+  Apache Kylinå¢å 
å¯¹Tableau 9åå¾®è½¯Excel, Power BIçæ¯æposted: Dec 25, 2015
 
   
 
@@ -270,13 +270,13 @@
 
 
 
-  Apache Kylin v1.1 
(incubating) æ£å¼åå¸posted: 
Oct 25, 2015
+  Apache Kylin v1.1 (incubating) 
Release Announcementposted: Oct 
25, 2015
 
   
 
 
 
-  Apache Kylin v1.1 (incubating) 
Release Announcementposted: Oct 
25, 2015
+  Apache Kylin v1.1 
(incubating) æ£å¼åå¸posted: 
Oct 25, 2015
 
   
 

Modified: kylin/site/development/howto_package.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_package.html?rev=1747361=1747360=1747361=diff
==
--- kylin/site/development/howto_package.html (original)
+++ kylin/site/development/howto_package.html Wed Jun  8 08:23:21 2016
@@ -478,7 +478,7 @@
 
 git 
clone https://github.com/apache/kylin kylin
 cd kylin
-./script/package.sh
+./build/script/package.sh
 
 In order to generate binary package, maven and 
npm are pre-requisites.
 

Modified: kylin/site/development/howto_release.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_release.html?rev=1747361=1747360=1747361=diff
==
--- kylin/site/development/howto_release.html (original)
+++ kylin/site/development/howto_release.html Wed Jun  8 08:23:21 2016
@@ -489,7 +489,7 @@
   Apache Kylin dist repo: https://dist.apache.org/repos/dist/dev/kylin;>https://dist.apache.org/repos/dist/dev/kylin
 
 
-Setup PGP signing keys
+Setup GPG signing keys
 Follow instructions at http://www.apache.org/dev/release-signing;>http://www.apache.org/dev/release-signing
 to create a key pair  
 Install gpg (On Mac OS X as sample):  
 brew install gpg
@@ -535,6 +535,7 @@ For example:  
 
   Set up signing keys as described above.
   Make sure you are using JDK 1.7 (not 1.8).
+  Make sure you are using GIT 2.7.2 or above.
   Make sure you are working on right release version number.
   Make sure that every âresolvedâ JIRA case (including duplicates) has 
a fix version assigned.
 
@@ -878,6 +879,18 @@ svn commit -m 'Remove o
 
 Release same version in JIRA, check https://issues.apache.org/jira/browse/KYLIN/?selectedTab=com.atlassian.jira.jira-projects-plugin:changelog-panel;>Change
 Log for the latest released version.
 
+Build and upload binary package
+After publish the release, you need generate the binary packages and then 
put them to the svn release repository;
+
+
+  Git checkout the tag for current release;
+  Make a binary package by refering to this 
doc;
+  Sign the generated binary package with gpg, e.g,: âgpg âarmor 
âoutput apa

kylin git commit: KYLIN-1693 add validation check

2016-06-08 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 6788ee6e1 -> 4a202d16d


KYLIN-1693 add validation check

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a202d16
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a202d16
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a202d16

Branch: refs/heads/master
Commit: 4a202d16d433314d1fb5cd90eaa9d4222a20f411
Parents: 6788ee6
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jun 8 15:05:38 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jun 8 16:23:38 2016 +0800

--
 .../model/validation/rule/FunctionRule.java | 28 +---
 1 file changed, 19 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a202d16/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
index 88564d9..e27a642 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
+import com.google.common.collect.Lists;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
@@ -39,15 +40,13 @@ import org.apache.kylin.metadata.model.ParameterDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 
 /**
- * Validate function parameter. Ticket:
- * https://github.scm.corp.ebay.com/Kylin/Kylin/issues/268
+ * Validate function parameter.
  * 
  * if type is column, check values are valid fact table columns if type is
  * constant, the value only can be numberic
  * 
  * the return type only can be int/bigint/long/double/decimal
  *
- * @author jianliu
  */
 public class FunctionRule implements IValidatorRule {
 
@@ -106,17 +105,28 @@ public class FunctionRule implements 
IValidatorRule {
 
 if 
(TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
 if (parameter.getNextParameter() == null) {
-context.addResult(ResultLevel.ERROR, "Must define 2 
parameters for function " + func.getExpression() + " in " + measure.getName());
+context.addResult(ResultLevel.ERROR, "Must define at least 
2 parameters for function " + func.getExpression() + " in " + 
measure.getName());
 return;
 }
 
-String embeded_groupby = 
parameter.getNextParameter().getValue();
-for (DimensionDesc dimensionDesc : cube.getDimensions()) {
-if (dimensionDesc.getColumn() != null && 
dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
-context.addResult(ResultLevel.ERROR, "Couldn't use 
column " + embeded_groupby + " in Top-N as it is already defined as a 
dimension.");
-return;
+ParameterDesc groupByCol = parameter.getNextParameter();
+List duplicatedCol = Lists.newArrayList();
+while (groupByCol != null) {
+String embeded_groupby = groupByCol.getValue();
+for (DimensionDesc dimensionDesc : cube.getDimensions()) {
+if (dimensionDesc.getColumn() != null && 
dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
+duplicatedCol.add(embeded_groupby);
+}
 }
+groupByCol = groupByCol.getNextParameter();
 }
+
+if (duplicatedCol.size() > 0) {
+context.addResult(ResultLevel.ERROR, "Couldn't use " + 
duplicatedCol.toString() + " in Top-N as it is already defined as dimension.");
+return;
+
+}
+
 }
 }

kylin git commit: KYLIN-1774 update tomcat version

2016-06-08 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 4a202d16d -> e7740c852


KYLIN-1774 update tomcat version

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7740c85
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7740c85
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7740c85

Branch: refs/heads/master
Commit: e7740c852e5db40e2c37527de2e99d81e3beb928
Parents: 4a202d1
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jun 8 16:39:17 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jun 8 16:39:17 2016 +0800

--
 build/script/download-tomcat.sh | 14 +++---
 server/pom.xml  |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e7740c85/build/script/download-tomcat.sh
--
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index ea2b86e..87467ce 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -27,22 +27,22 @@ if [[ `uname -a` =~ "Darwin" ]]; then
 alias md5cmd="md5 -q"
 fi
 
-if [ ! -f "build/apache-tomcat-7.0.59.tar.gz" ]
+if [ ! -f "build/apache-tomcat-7.0.69.tar.gz" ]
 then
 echo "no binary file found"
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.59/bin/apache-tomcat-7.0.59.tar.gz
 || echo "download tomcat failed"
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.69/bin/apache-tomcat-7.0.69.tar.gz
 || echo "download tomcat failed"
 else
-if [ `md5cmd build/apache-tomcat-7.0.59.tar.gz | awk '{print $1}'` != 
"ec570258976edf9a833cd88fd9220909" ]
+if [ `md5cmd build/apache-tomcat-7.0.69.tar.gz | awk '{print $1}'` != 
"10a071e5169a1a8b14ff35a0ad181052" ]
 then
 echo "md5 check failed"
-rm build/apache-tomcat-7.0.59.tar.gz
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.59/bin/apache-tomcat-7.0.59.tar.gz
 || echo "download tomcat failed"
+rm build/apache-tomcat-7.0.69.tar.gz
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.69/bin/apache-tomcat-7.0.69.tar.gz
 || echo "download tomcat failed"
 fi
 fi
 unalias md5cmd
 
-tar -zxvf build/apache-tomcat-7.0.59.tar.gz -C build/
-mv build/apache-tomcat-7.0.59 build/tomcat
+tar -zxvf build/apache-tomcat-7.0.69.tar.gz -C build/
+mv build/apache-tomcat-7.0.69 build/tomcat
 rm -rf build/tomcat/webapps/*
 
 mv build/tomcat/conf/server.xml build/tomcat/conf/server.xml.bak

http://git-wip-us.apache.org/repos/asf/kylin/blob/e7740c85/server/pom.xml
--
diff --git a/server/pom.xml b/server/pom.xml
index bb70808..3909d79 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -411,19 +411,19 @@
 
 org.apache.tomcat
 tomcat-catalina
-7.0.52
+7.0.69
 provided
 
 
 org.apache.tomcat
 tomcat-jasper
-7.0.52
+7.0.69
 provided
 
 
 org.apache.tomcat.embed
 tomcat-embed-core
-7.0.52
+7.0.69
 provided

svn commit: r13934 - in /release/kylin/apache-kylin-1.5.2.1: apache-kylin-1.5.2.1-HBase1.x-bin.tar.gz.asc apache-kylin-1.5.2.1-bin.tar.gz.asc apache-kylin-1.5.2.1-cdh5.7-bin.tar.gz.asc

2016-06-08 Thread shaofengshi

Author: shaofengshi
Date: Wed Jun  8 07:33:46 2016
New Revision: 13934

Log:
add signature files for binary packages

Added:

release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-HBase1.x-bin.tar.gz.asc
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-bin.tar.gz.asc

release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-cdh5.7-bin.tar.gz.asc

Added: 
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-HBase1.x-bin.tar.gz.asc
==
--- 
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-HBase1.x-bin.tar.gz.asc 
(added)
+++ 
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-HBase1.x-bin.tar.gz.asc 
Wed Jun  8 07:33:46 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJXV8lrAAoJEKvJ+IVvz0cgQs4QAKqxqpgffgbwRKlg75OBQh12
+fEfhUq4+WpgubfCLT67t2k+4xzrkZRfAprsUDURaCvVkjVdoQtbKOhQmiyl+lgDx
+eVmE0e+oqldNH3j6KmQ/VOLSs2thq6Bu18xb7EcDwdFxULJbvujmxVekmPQoHjqq
+tc50580oMv5L7G07pIqmLxsSsyYPz0fN/aQuYULyuQssksgqlSD7i3+ntDvc2oex
+cPZgMOV98TCYu9H7m2bcIFW3QS2b5CYdxgi1nuYV4zJu4l7C8cPPReUWQ/zh/Cuc
+QjFPAhM5hMHStUnmtd3aPEDjzfWtARZr6w3dtn/9dK1QNQUIvKpoZ2mFFz0U0y9e
+PeNJY6I5Byg9GJzoyd/gER9IxyMsdS5LFGghBNb1BAKcOLjWA79h0N72IGkBgl6z
+kjBnCELj1Vi8zWICt7YgfdW/reSRO1aHIL0gQ/TdU1+r4fLD+bn12X5qzWVnWJi5
+wihK+9Fovt3CKbYJH8ELoQc1Sw5JXdsgHm1awgeRQv/wolAll2N61iuO+a0+CMUz
+tIkvW7bwcNt2Y4sBgmwAf9CgFqwmD4m0DPEWeKuvTJ+oB+EQxJ7gW5BZ7emB4IrY
+/kYoKp2NkBpj41o273tPFlEWsY7/UJBAvSgDDk6xcAJuFzkxNxHgGXqbsajbNlC/
+DshjA+BauDfL3pDlxNx+
+=1Z3r
+-END PGP SIGNATURE-

Added: release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-bin.tar.gz.asc
==
--- release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-bin.tar.gz.asc 
(added)
+++ release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-bin.tar.gz.asc Wed 
Jun  8 07:33:46 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJXV8iHAAoJEKvJ+IVvz0cgHCkP/inuBF5ynmE/imohVAwk001l
+j6+DNo9A6RGddAJg9yWEX7/JnASdnMOcwT1CxQpThzwCFof9IXa1BGMESDWo6Bfk
+K2NaFQeE39/2vkMOSbycduh0765XemhYy9RcdbM9GQikF71ONX9q6P80ho+x22F1
+0zUEykzj32HoarnKjHgs7G1DVuNRXe7fJTfJC2aFvecmUVmzYeV36C+F6zVlTymQ
+LKbCoxUoG2Q/N1KMisAVqmy7IqO/n9whH9Pqo7aDxNTCSsoEkVzFaTR/63JAmz5C
+0I4FjHvcUKRH54uxk0UIxB1nlTQGG0TRYX0diwBIASs1aNSTRItBpkBAxkMMrjXR
+/2YVTH0HZA/qZTFoRNQ6snAUAdmLRGDSUvaxO7etujhn577/9MJe+o9fhjRmKsO3
+UyKLS8Mf4farubfzOYZi2P/MiN6Ak4/BwuWmAnCQ8oG30tAFxvO4gz6w1laluc/N
+Luts6SzreeN6Okbn1v5YtoxYocBzKVoml7kISdSZtCRZqIMOiYd3osrbfNI5n/xC
+gTAlHqMvsEoNG9ie7DIFKeUCrc0xSzNQw+vumVxieoDMHhTa509MGiSjPpHDcczT
+DpUpfSQWD//hU19uB71h/fPHkm5MO3Vd+CKLsJqh6wWLi4gg14XRFQ6JCR1Kt2kn
+5atzweua34LpNFe18NIi
+=f5YL
+-END PGP SIGNATURE-

Added: 
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-cdh5.7-bin.tar.gz.asc
==
--- 
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-cdh5.7-bin.tar.gz.asc 
(added)
+++ 
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-cdh5.7-bin.tar.gz.asc 
Wed Jun  8 07:33:46 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJXV8j7AAoJEKvJ+IVvz0cgHuoP/jREIuw3vimK25zc8Fs86iX2
+yLswhyQgmpLmwwNPZWXi/N4Heg83CVG8BGld/J2opFuAzx3QufOIxYZwD4Fbs+//
+4hBuDP9gulEHukdn8DYLNv8/W3lOHe6CS2Hz/xUfPjBklocBZ1E9iOkKnddJnD4b
+WKxt/ElbrrhDD43f/7z37pCfv/FGgH5q4hElG4ClqRk36hmXa96lW+7jnkz8myvX
+vSFb+l88kbwKNjAHL5t7f87Oj+MswEzU2+G41sV6T3S4d6qMvyItT6tHCqI/05lW
+fOjFE/jh127m/DvO8SvyYR2wESCEWh0ybjcqcarafxcgGuxAz+tj8nNnPH7bLK5L
+Zx28ZZVl0sjw3/aMFIP9WXO8uBG5m3Y5wIS/qIeBDZCIvckfEUYg+s0QqNMSzKXK
+pY5ul1wpkorNqF4/9lVGyVj8Dx+I7XXiGtenTXIoZOcKdXV+81TIRI1ALBQ3i/7Z
+NNPlDQPesaUmmc7h6oojt0rQcFA5SMSbR7I1Wdg+ehx+iXTPvZV3zN6sRQNhym6t
+4YIQFwegcbTaM5YKAOWO1u3SZCDTzMDOwgGVItpAfULZvO9gOG/ngf6K+SSzJSSz
+WZ2n19+1NcTE21I4WP9roKwX9Q1CmISS7ChEBbJ8iyf36zlZDrvzQ/AmKrYMtj/4
+wAQbBCi3aRKmq9IsLCl1
+=s7f6
+-END PGP SIGNATURE-

svn commit: r13923 - in /release/kylin/apache-kylin-1.5.2.1: apache-kylin-1.5.2.1-HBase1.x-bin.tar.gz apache-kylin-1.5.2.1-HBase1.x.bin.tar.gz

2016-06-07 Thread shaofengshi

Author: shaofengshi
Date: Tue Jun  7 14:23:58 2016
New Revision: 13923

Log:
fix wrong file name

Added:
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-HBase1.x-bin.tar.gz
  - copied unchanged from r13922, 
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-HBase1.x.bin.tar.gz
Removed:
release/kylin/apache-kylin-1.5.2.1/apache-kylin-1.5.2.1-HBase1.x.bin.tar.gz

kylin git commit: update download page for v1.5.2.1

2016-06-07 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document 19138dcd5 -> b918189a4


update download page for v1.5.2.1


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b918189a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b918189a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b918189a

Branch: refs/heads/document
Commit: b918189a44f2f3e8512c16459fb699f89bd1558a
Parents: 19138dc
Author: shaofengshi <shaofeng...@apache.org>
Authored: Tue Jun 7 22:25:47 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jun 7 22:25:47 2016 +0800

--
 website/_docs15/release_notes.md | 11 +++
 website/download/index.cn.md | 24 
 website/download/index.md| 16 
 3 files changed, 31 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b918189a/website/_docs15/release_notes.md
--
diff --git a/website/_docs15/release_notes.md b/website/_docs15/release_notes.md
index e212373..19094be 100644
--- a/website/_docs15/release_notes.md
+++ b/website/_docs15/release_notes.md
@@ -15,6 +15,17 @@ or send to Apache Kylin mailing list:
 * User relative: [u...@kylin.apache.org](mailto:u...@kylin.apache.org)
 * Development relative: [d...@kylin.apache.org](mailto:d...@kylin.apache.org)
 
+## v1.5.2.1 - 2016-06-07
+_Tag:_ [kylin-1.5.2.1](https://github.com/apache/kylin/tree/kylin-1.5.2.1)
+
+This is a hot-fix version on v1.5.2, no new feature introduced, please upgrade 
to this version;
+
+__Bug__
+
+* [KYLIN-1758] - createLookupHiveViewMaterializationStep will create 
intermediate table for fact table
+* [KYLIN-1739] - kylin_job_conf_inmem.xml can impact non-inmem MR job
+
+
 ## v1.5.2 - 2016-05-26
 _Tag:_ [kylin-1.5.2](https://github.com/apache/kylin/tree/kylin-1.5.2)
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b918189a/website/download/index.cn.md
--
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index 7060250..f60496a 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -3,31 +3,31 @@ layout: download-cn
 title: ä¸è½½
 ---
 
-__1.5.2 ææ°åå¸(æºä»£ç )__
+__ææ°åå¸(æºä»£ç )__
 
 ææ°åå¸çApache Kylinâ¢å¯ä»¥ä»ASFç½ç«ä¸è½½ï¼:
 
-* [Apache Kylin 
v1.5.2](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.5.2/)
+* [Apache Kylin 
v1.5.2.1](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.5.2.1/)
 * [åå¸æ¥å¿](http://kylin.apache.org/docs15/release_notes.html)
-* Git æ ç¾: [kylin-1.5.2](https://github.com/apache/kylin/tree/kylin-1.5.2)
-* Git Commit: 
[af2646b72fbb6dc81699ad6661303fd612a2eebf](https://github.com/apache/kylin/commit/af2646b72fbb6dc81699ad6661303fd612a2eebf)
+* Git æ ç¾: 
[kylin-1.5.2.1](https://github.com/apache/kylin/tree/kylin-1.5.2.1)
+* Git Commit: 
[71a847bbe97ec4e6f23b1bf271132d6ff9955a96](https://github.com/apache/kylin/commit/71a847bbe97ec4e6f23b1bf271132d6ff9955a96)
 
-__1.5.2 äºè¿å¶å for HBase 0.98/0.99__
+__1.5.2.1 äºè¿å¶å for HBase 0.98/0.99__
 
 ä¸ºæ¹ä¾¿ä½¿ç¨ï¼æä»¬æä¾é¢æåçäºè¿å¶å®è£åï¼
 
-* 
[apache-kylin-1.5.2-bin.tar.gz](https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.5.1/apache-kylin-1.5.2-bin.tar.gz)
+* 
[apache-kylin-1.5.2.1-bin.tar.gz](https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.5.1/apache-kylin-1.5.2.1-bin.tar.gz)
 * [å®è£å¸®å©](http://kylin.apache.org/docs15/install)
 
-__1.5.2 äºè¿å¶å for HBase 1.x__
+__1.5.2.1 äºè¿å¶å for HBase 1.x__
 
-* 
[apache-kylin-1.5.2-HBase1.x-bin.tar.gz](https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.5.1/apache-kylin-1.5.2-HBase1.x-bin.tar.gz)
-* Git commit 
[e85b49823e3cfea027cca497e898064fbefbd449](https://github.com/apache/kylin/commit/e85b49823e3cfea027cca497e898064fbefbd449)
+* 
[apache-kylin-1.5.2.1-HBase1.x-bin.tar.gz](https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.5.1/apache-kylin-1.5.2.1-HBase1.x-bin.tar.gz)
+* Git commit 
[cf4d2940b67d622eacd2ac9a913b221091a35c2e](https://github.com/apache/kylin/commit/cf4d2940b67d622eacd2ac9a913b221091a35c2e)
 
-__1.5.2 äºè¿å¶å for CDH 5.7__
+__1.5.2.1 äºè¿å¶å for CDH 5.7__
 
-* 
[apache-kylin-1.5.2-cdh5.7-bin.tar.gz](https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.5.1/apache-kylin-1.5.2-cdh5.7-bin.tar.gz)
-* Git commit 
[e0fbb3417063fe33b99e82724e19e107787b2e62](https://github.com/apache/kylin/commit/e0fbb3417063fe33b99e82724e19e107787b2e62)
+* 
[apache-kylin-1.5.2.1-cdh5.7-bin.tar.gz](https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.5.1/apache-kylin-1.5.2.1-cdh5.7-bin.tar.gz

kylin git commit: defer to publish yerui's blog on the count distinct

2016-06-10 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document bc806da7c -> d52333879


defer to publish yerui's blog on the count distinct


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5233387
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5233387
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5233387

Branch: refs/heads/document
Commit: d523338793933aa6e60fbe0e5cc63884dfbf2abf
Parents: bc806da
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sat Jun 11 08:55:39 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sat Jun 11 08:55:39 2016 +0800

--
 .../blog/2016-06-10-count-distinct-in-kylin.md  | 96 
 1 file changed, 96 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d5233387/website/_posts/blog/2016-06-10-count-distinct-in-kylin.md
--
diff --git a/website/_posts/blog/2016-06-10-count-distinct-in-kylin.md 
b/website/_posts/blog/2016-06-10-count-distinct-in-kylin.md
deleted file mode 100644
index dbf0862..000
--- a/website/_posts/blog/2016-06-10-count-distinct-in-kylin.md
+++ /dev/null
@@ -1,96 +0,0 @@

-layout: post-blog
-title:  Use Count Distinct in Apache Kylin
-date:   2016-06-10 18:30:00
-author: Yerui Sun 
-categories: blog

-
-Since v.1.5.3
-
-## Background
-Count Distinct is a commonly measure in OLAP analyze, usually used for uv, 
etc. Apache Kylin offers two kinds of count distinct, approximately and 
precisely, differs on resource and performance.
-
-## Approximately Count Distinct
-Apache Kylin implements approximately count distinct using HyperLogLog 
algorithm, offered serveral precision, with the error rates from 9.75% to 
1.22%. 
-The result of measure has theorically upper limit in size, as 2^N bytes. For 
the max precision N=16, the upper limit is 64KB, and the max error rate is 
1.22%. 
-This implementation's pros is fast caculating and storage resource saving, but 
can't be used for precisely requirements.
-
-## Precisely Count Distinct
-Apache Kylin also implements precisely count distinct based on bitmap. For the 
data with type tiny int(byte), small int(short) and int, project the value into 
the bitmap directly. For the data with type long, string and others, encode the 
value as String into a dict, and project the dict id into the bitmap.
-The result of measure is the serialized data of bitmap, not just the count 
value. This makes sure that the rusult is always right with any roll-up, even 
across segments.
-This implementation's pros is precesily result, without error, but needs more 
storage resources. One result size maybe hundreds of MB, when the count 
distinct value over millions.
-
-## Global Dictionary
-Apache Kylin encode value into dictionay at the segment level by default. That 
means one same value in different segments maybe encoded into different id, 
which means the result of precisely count distinct maybe not correct.
-We introduced Global Dictionary with ensurance that one same value always 
encode into same id in different segments, to resolve this problem. Meanwhile, 
the capacity of dict has expanded dramatically, upper to support 2G values in 
one dict. It can also be used to replace default dictionary which has 5M values 
limitation.
-Current version has no UI for global dictionary yet, and the cube desc json 
shoule be modified to enable it:
-
-```
-"dictionaries": [
-{
-  "column": "SUCPAY_USERID",
- "reuse": "USER_ID",
-  "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
-}
-]
-```
-
-The `column` means the column which to be encoded, the `builder` specifies the 
dictionary builder, only `org.apache.kylin.dict.GlobalDictionaryBuilder` is 
available for now.
-The 'reuse` is used to optimize the dict of more than one columns based on one 
dataset, please refer the next section 'Example' for more details.
-The global dictionay can't be used for dimensiion encoding for now, that means 
if one column is used for dimension and count distinct measure in one cube, the 
dimension encoding should be others but not dict.
-
-## Example
-Here's some example data:
-| DT   | USER\_ID | FLAG1 | FLAG2 | USER\_ID\_FLAG1 | USER\_ID\_FLAG2 |
-| :--: | :--: | :---: | :---: | :-: | :-: |
-| 2016-06-08   | AAA  | 1 | 1 | AAA | AAA |
-| 2016-06-08   | BBB  | 1 | 1 | BBB | BBB |
-| 2016-06-08   | CCC  | 0 | 1 | NULL| CCC |
-| 2016-06-09   | AAA  | 0 | 1 | NULL| AAA |
-| 2016-06-09   | CCC  | 1

[4/4] kylin git commit: KYLIN-1758 add the hive set statements

2016-06-04 Thread shaofengshi

KYLIN-1758 add the hive set statements


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1d17e21
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1d17e21
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1d17e21

Branch: refs/heads/v1.5.2.1-release-rc1
Commit: a1d17e215561089bb5958b84e876a74fc60f092f
Parents: ef1d13f
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sat Jun 4 16:22:38 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sat Jun 4 16:22:38 2016 +0800

--
 .../org/apache/kylin/job/JoinedFlatTable.java   | 56 +++-
 .../apache/kylin/source/hive/HiveMRInput.java   | 14 +
 2 files changed, 44 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a1d17e21/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 5886325..f13a48c 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -55,6 +55,36 @@ public class JoinedFlatTable {
 return storageDfsDir + "/" + intermediateTableDesc.getTableName();
 }
 
+public static String generateHiveSetStatements(JobEngineConfig 
engineConfig) throws IOException {
+StringBuilder buffer = new StringBuilder();
+File hadoopPropertiesFile = new 
File(engineConfig.getHiveConfFilePath());
+
+if (hadoopPropertiesFile.exists()) {
+DocumentBuilderFactory factory = 
DocumentBuilderFactory.newInstance();
+DocumentBuilder builder;
+Document doc;
+try {
+builder = factory.newDocumentBuilder();
+doc = builder.parse(hadoopPropertiesFile);
+NodeList nl = doc.getElementsByTagName("property");
+for (int i = 0; i < nl.getLength(); i++) {
+String name = 
doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
+String value = 
doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
+if (!name.equals("tmpjars")) {
+buffer.append("SET " + name + "=" + value + ";\n");
+}
+}
+
+} catch (ParserConfigurationException e) {
+throw new IOException(e);
+} catch (SAXException e) {
+throw new IOException(e);
+}
+}
+
+return buffer.toString();
+}
+
 public static String generateCreateTableStatement(IJoinedFlatTableDesc 
intermediateTableDesc, String storageDfsDir) {
 StringBuilder ddl = new StringBuilder();
 
@@ -86,32 +116,6 @@ public class JoinedFlatTable {
 
 public static String generateInsertDataStatement(IJoinedFlatTableDesc 
intermediateTableDesc, JobEngineConfig engineConfig) throws IOException {
 StringBuilder sql = new StringBuilder();
-
-File hadoopPropertiesFile = new 
File(engineConfig.getHiveConfFilePath());
-
-if (hadoopPropertiesFile.exists()) {
-DocumentBuilderFactory factory = 
DocumentBuilderFactory.newInstance();
-DocumentBuilder builder;
-Document doc;
-try {
-builder = factory.newDocumentBuilder();
-doc = builder.parse(hadoopPropertiesFile);
-NodeList nl = doc.getElementsByTagName("property");
-for (int i = 0; i < nl.getLength(); i++) {
-String name = 
doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
-String value = 
doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
-if (name.equals("tmpjars") == false) {
-sql.append("SET " + name + "=" + value + 
";").append("\n");
-}
-}
-
-} catch (ParserConfigurationException e) {
-throw new IOException(e);
-} catch (SAXException e) {
-throw new IOException(e);
-}
-}
-
 sql.append("INSERT OVERWRITE TABLE " + 
intermediateTableDesc.getTableName() + " " + 
generateSelectDataStatement(intermediateTableDesc) + ";").append("\n");
 
 return sql.toString();

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1d17e21/source-hiv

[3/4] kylin git commit: update version number

2016-06-04 Thread shaofengshi

update version number


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef1d13f8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef1d13f8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef1d13f8

Branch: refs/heads/v1.5.2.1-release-rc1
Commit: ef1d13f8482b755b59c077c5b1ae1898e2e05e66
Parents: 6688d64
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sat Jun 4 08:24:38 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sat Jun 4 08:24:38 2016 +0800

--
 assembly/pom.xml | 2 +-
 atopcalcite/pom.xml  | 2 +-
 core-common/pom.xml  | 2 +-
 core-cube/pom.xml| 2 +-
 core-dictionary/pom.xml  | 2 +-
 core-job/pom.xml | 2 +-
 core-metadata/pom.xml| 2 +-
 core-storage/pom.xml | 2 +-
 engine-mr/pom.xml| 2 +-
 engine-spark/pom.xml | 2 +-
 engine-streaming/pom.xml | 2 +-
 invertedindex/pom.xml| 2 +-
 jdbc/pom.xml | 2 +-
 kylin-it/pom.xml | 2 +-
 pom.xml  | 4 ++--
 query/pom.xml| 2 +-
 server/pom.xml   | 2 +-
 source-hive/pom.xml  | 2 +-
 source-kafka/pom.xml | 2 +-
 storage-hbase/pom.xml| 2 +-
 tool/pom.xml | 2 +-
 21 files changed, 22 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index b6e3675..204035b 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.5.2
+1.5.2.1
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 7c70df3..7779201 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 7647621..5030d5a 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 2309872..f4e0c71 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 1af0091..a6cd19c 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 864bf1b..383303c 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index f65129d..184dc5a 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 2f08dd9..4948e59 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef1d13f8/engine-mr/pom.xml
--
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 379eaa6..dcd3e2f 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-

kylin git commit: [maven-release-plugin] prepare release v1.5.2.1-release

2016-06-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.5.2.1-release 404c5305b -> 002bc8dea


[maven-release-plugin] prepare release v1.5.2.1-release


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/002bc8de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/002bc8de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/002bc8de

Branch: refs/heads/v1.5.2.1-release
Commit: 002bc8dea341a7a1bef51d6de799deae353182c2
Parents: 404c530
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sat Jun 4 10:09:15 2016 +
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sat Jun 4 10:09:15 2016 +

--
 assembly/pom.xml | 2 +-
 atopcalcite/pom.xml  | 2 +-
 core-common/pom.xml  | 2 +-
 core-cube/pom.xml| 2 +-
 core-dictionary/pom.xml  | 2 +-
 core-job/pom.xml | 2 +-
 core-metadata/pom.xml| 2 +-
 core-storage/pom.xml | 2 +-
 engine-mr/pom.xml| 2 +-
 engine-spark/pom.xml | 2 +-
 engine-streaming/pom.xml | 2 +-
 invertedindex/pom.xml| 2 +-
 jdbc/pom.xml | 2 +-
 kylin-it/pom.xml | 2 +-
 pom.xml  | 4 ++--
 query/pom.xml| 2 +-
 server/pom.xml   | 2 +-
 source-hive/pom.xml  | 2 +-
 source-kafka/pom.xml | 2 +-
 storage-hbase/pom.xml| 2 +-
 tool/pom.xml | 2 +-
 21 files changed, 22 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/002bc8de/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 2aa8c87..204035b 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/002bc8de/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index ec23344..7779201 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/002bc8de/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 381192a..5030d5a 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/002bc8de/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0fa5ffe..f4e0c71 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/002bc8de/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 56d70e7..a6cd19c 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/002bc8de/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 8a0ee7d..383303c 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/002bc8de/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 76d709c..184dc5a 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/002bc8de/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index b9daa31..4948e59 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1-SNAPSHOT
+1.5.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/002bc8de/engine-mr/pom.xml

kylin git commit: KYLIN-1763 correct the version number for release

2016-06-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.5.2.1-release-rc1 a1d17e215 -> 404c5305b


KYLIN-1763 correct the version number for release

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/404c5305
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/404c5305
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/404c5305

Branch: refs/heads/v1.5.2.1-release-rc1
Commit: 404c5305bfd6a5583e932a3116774a21a1a544e1
Parents: a1d17e2
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sat Jun 4 17:03:07 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sat Jun 4 17:03:07 2016 +0800

--
 assembly/pom.xml | 2 +-
 atopcalcite/pom.xml  | 2 +-
 core-common/pom.xml  | 2 +-
 core-cube/pom.xml| 2 +-
 core-dictionary/pom.xml  | 2 +-
 core-job/pom.xml | 2 +-
 core-metadata/pom.xml| 2 +-
 core-storage/pom.xml | 2 +-
 engine-mr/pom.xml| 2 +-
 engine-spark/pom.xml | 2 +-
 engine-streaming/pom.xml | 2 +-
 invertedindex/pom.xml| 2 +-
 jdbc/pom.xml | 2 +-
 kylin-it/pom.xml | 2 +-
 pom.xml  | 2 +-
 query/pom.xml| 2 +-
 server/pom.xml   | 2 +-
 source-hive/pom.xml  | 2 +-
 source-kafka/pom.xml | 2 +-
 storage-hbase/pom.xml| 2 +-
 tool/pom.xml | 2 +-
 21 files changed, 21 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/404c5305/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 204035b..2aa8c87 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.5.2.1
+1.5.2.1-SNAPSHOT
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/404c5305/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 7779201..ec23344 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1
+1.5.2.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/404c5305/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 5030d5a..381192a 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1
+1.5.2.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/404c5305/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index f4e0c71..0fa5ffe 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1
+1.5.2.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/404c5305/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index a6cd19c..56d70e7 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1
+1.5.2.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/404c5305/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 383303c..8a0ee7d 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1
+1.5.2.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/404c5305/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 184dc5a..76d709c 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1
+1.5.2.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/404c5305/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 4948e59..b9daa31 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.2.1
+1.5.2.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/404c5305/engine-mr/pom.xml
--

svn commit: r1747951 - in /kylin/site: ./ blog/ blog/2016/06/ blog/2016/06/10/ blog/2016/06/10/diagnosis-tool/ blog/2016/06/10/standalone-hbase-cluster/ development/

2016-06-11 Thread shaofengshi

Author: shaofengshi
Date: Sun Jun 12 03:19:22 2016
New Revision: 1747951

URL: http://svn.apache.org/viewvc?rev=1747951=rev
Log:
update blog

Added:
kylin/site/blog/2016/06/
kylin/site/blog/2016/06/10/
kylin/site/blog/2016/06/10/diagnosis-tool/
kylin/site/blog/2016/06/10/diagnosis-tool/index.html
kylin/site/blog/2016/06/10/standalone-hbase-cluster/
kylin/site/blog/2016/06/10/standalone-hbase-cluster/index.html
Modified:
kylin/site/blog/index.html
kylin/site/development/howto_release.html
kylin/site/feed.xml

Added: kylin/site/blog/2016/06/10/diagnosis-tool/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/blog/2016/06/10/diagnosis-tool/index.html?rev=1747951=auto
==
--- kylin/site/blog/2016/06/10/diagnosis-tool/index.html (added)
+++ kylin/site/blog/2016/06/10/diagnosis-tool/index.html Sun Jun 12 03:19:22 
2016
@@ -0,0 +1,269 @@
+
+
+
+   
+
+
+  
+  
+  
+
+  Apache Kylin | Diagnosis Tool Introduction
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+  
+  
+
+  http://kylin.apache.org/blog/2016/06/10/diagnosis-tool/;>
+  http://kylin.apache.org/feed.xml; />
+
+
+
+  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new 
Date();a=s.createElement(o),
+  
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+  //oringal tracker for kylin.io
+  ga('create', 'UA-55534813-1', 'auto');
+  //new tracker for kylin.apache.org
+  ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'});
+
+  ga('send', 'pageview');
+  ga('toplevel.send', 'pageview');
+
+
+
+
+ 
+ 
+
+
+   
+   
+
+
+  
+  
+  Apache Kylinâ¢ Extreme OLAP Engine for Big 
Data 
+
+  
+  
+
+  
+  
+  
+
+
+
+  
+Toggle navigation
+
+
+
+  
+ 
+
+
+
+
+  
+ Home
+  Docs
+  Download
+  Community
+  Development
+  Blog
+  ä¸æç  
+  https://twitter.com/apachekylin; target="_blank" 
class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" >
+  https://github.com/apache/kylin; target="_blank" 
class="fa fa-github-alt fa-lg" title="Github: apache/kylin" >  
+  https://www.facebook.com/kylinio; target="_blank" 
class="fa fa-facebook fa-lg" title="Facebook: kylin.io" >   
+
+
+  
+
+ 
+
+   
+   
+   
+   Apache Kylinâ¢ 
Technical Blog
+   
+   
+
+   
+   
+
+   
+
+
+
+  
+Diagnosis Tool Introduction
+Jun 10, 2016 â¢ Dong Li
+  
+
+  
+Introduction
+
+Since Apache Kylin 1.5.2, thereâs a diagnosis tool on Web UI, which aims 
to help Kylin admins to extract diagnostic information for fault analysis and 
performance tunning.
+
+Project Diagnosis
+When user met issues about query failure, slow queries, metadata management 
and so on, he could click the âDiagnosisâ button on System tabpage.
+
+
+
+Several seconds later, a diagnosis package will be avaliable to download 
from web browser, which contains kylin logs, metadata, configuration etc. Users 
could extract the package and analyze. Also when users asking help from experts 
in his team, attaching the package would raise the communication 
effeiciency.
+
+Job Diagnosis
+When user met issues about jobs, such as cubing failed, slow job and so on, 
he could click the âDiagnosisâ button in the Jobâs Action menu.
+
+
+
+The same with Project Diagnosis, a diagnosis package will be downloaded 
from web browser, which contains related logs, MR job info, metadata etc. User 
could use it for analysis or ask for help.
+
+  
+
+
+
+
+
+
+
+   
+   
+ 
+   
+
+
+
+
+
+
+http://www.apache.org;>
+
+
+
+
+The contents of this website are Â© 2015 Apache 
Software Foundation under the terms of the http://www.apache.org/licenses/LICENSE-2.0;> 
Apache License v2 . Apache Kylin and
+its logo are trademarks of the Apache Software 
Foundation.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+   
+   
+
+
+
+
+

Added: kylin/site/blog/2016/06/10/standalone-hbase-cluster/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/blog/2016/06/10/standalone-hbase-cluster/index

kylin git commit: fix a syntax issue in 2016-06-10-standalone-hbase-cluster.md

2016-06-11 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document d52333879 -> 561532ba5


fix a syntax issue in 2016-06-10-standalone-hbase-cluster.md


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/561532ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/561532ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/561532ba

Branch: refs/heads/document
Commit: 561532ba5e2f0ee71a367e0d66f99dc8f39b7a84
Parents: d523338
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sun Jun 12 11:18:43 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sun Jun 12 11:18:43 2016 +0800

--
 website/_posts/blog/2016-06-10-standalone-hbase-cluster.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/561532ba/website/_posts/blog/2016-06-10-standalone-hbase-cluster.md
--
diff --git a/website/_posts/blog/2016-06-10-standalone-hbase-cluster.md 
b/website/_posts/blog/2016-06-10-standalone-hbase-cluster.md
index 35e8604..0ee1f08 100644
--- a/website/_posts/blog/2016-06-10-standalone-hbase-cluster.md
+++ b/website/_posts/blog/2016-06-10-standalone-hbase-cluster.md
@@ -1,10 +1,10 @@
---
+---
 layout: post-blog
 title:  Deploy Apache Kylin with Standalone HBase Cluster
 date:   2016-06-10 17:30:00
 author: Yerui Sun
 categories: blog
---
+---
 
 ## Introduction

svn commit: r1747953 - in /kylin/site: blog/2016/06/10/standalone-hbase-cluster/index.html feed.xml

2016-06-11 Thread shaofengshi

Author: shaofengshi
Date: Sun Jun 12 03:33:44 2016
New Revision: 1747953

URL: http://svn.apache.org/viewvc?rev=1747953=rev
Log:
update format

Modified:
kylin/site/blog/2016/06/10/standalone-hbase-cluster/index.html
kylin/site/feed.xml

Modified: kylin/site/blog/2016/06/10/standalone-hbase-cluster/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/blog/2016/06/10/standalone-hbase-cluster/index.html?rev=1747953=1747952=1747953=diff
==
--- kylin/site/blog/2016/06/10/standalone-hbase-cluster/index.html (original)
+++ kylin/site/blog/2016/06/10/standalone-hbase-cluster/index.html Sun Jun 12 
03:33:44 2016
@@ -191,12 +191,15 @@
 Apache Kylin mainly use HBase to storage cube data. The performance of 
HBase cluster impacts on the query performance of Kylin directly. In common 
scenario, HBase is deployed with MR/Hive on one HDFS cluster, which makes that 
the resouces HBase used is limited, and the MR job affects the performance of 
HBase. These problems can be resolved with standalone HBase cluster, and Apache 
Kylin has support this deploy mode for now.
 
 Enviroment Requirements
-To enable standalone HBase cluster supporting, check the basic enviroments 
at first:
- - Deploy the main cluster and hbase cluster, make sure both works normally
- - Make sure Kylin Server can access both clusters using hdfs shell with fully 
qualifiered path
- - Make sure Kylin Server can submit MR job to main cluster, and can use hive 
shell to access data warehouse, make sure the configurations of hadoop and hive 
points to main cluster
- - Make sure Kylin Server can access hbase cluster using hbase shell, make 
sure the configuration of hbase points to hbase cluster
- - Make sure the job on main cluster can access hbase cluster directly
+To enable standalone HBase cluster supporting, check the basic enviroments 
at first:
+
+
+  Deploy the main cluster and hbase cluster, make sure both works 
normally
+  Make sure Kylin Server can access both clusters using hdfs shell with 
fully qualifiered path
+  Make sure Kylin Server can submit MR job to main cluster, and can use 
hive shell to access data warehouse, make sure the configurations of hadoop and 
hive points to main cluster
+  Make sure Kylin Server can access hbase cluster using hbase shell, make 
sure the configuration of hbase points to hbase cluster
+  Make sure the job on main cluster can access hbase cluster directly
+
 
 Configurations
 Update the config kylin.hbase.cluster.fs in kylin.properties, 
with a value as the Namenode address of HBase Cluster, like hdfs://hbase-cluster-nn01.example.com:8020
@@ -204,9 +207,12 @@
 Notice that the value should keep consistent with the Namenode address of 
root.dir on HBase Master node, to ensure 
bulkload into hbase.
 
 Using NN HA
-HDFS Namenode HA improved the availablity of cluster significantly, and 
maybe the HBase cluster enabled it. Apache Kylin doesnât support the HA 
perfectly for now, and hereâs the workaroud:
- - Add all dfs.nameservices related 
configs of HBase Cluster into hadoop/etc/hadoop/hdfs-site.xml in Kylin 
Server, to make sure that can access HBase Cluster using hdfs shell with 
nameservice path
- - Add all dfs.nameservices related 
configs of both two clusters into kylin_job_conf.xml, to make sure that the MR 
job can access hbase cluster with nameservice path
+HDFS Namenode HA improved the availablity of cluster significantly, and 
maybe the HBase cluster enabled it. Apache Kylin doesnât support the HA 
perfectly for now, and hereâs the workaroud:
+
+
+  Add all dfs.nameservices related 
configs of HBase Cluster into hadoop/etc/hadoop/hdfs-site.xml in Kylin 
Server, to make sure that can access HBase Cluster using hdfs shell with 
nameservice path
+  Add all dfs.nameservices related 
configs of both two clusters into kylin_job_conf.xml, to make sure that the MR 
job can access hbase cluster with nameservice path
+
 
 TroubleShooting
 

Modified: kylin/site/feed.xml
URL: 
http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1747953=1747952=1747953=diff
==
--- kylin/site/feed.xml (original)
+++ kylin/site/feed.xml Sun Jun 12 03:33:44 2016
@@ -19,8 +19,8 @@
 Apache Kylin Home
 http://kylin.apache.org/
 http://kylin.apache.org/feed.xml; rel="self" 
type="application/rss+xml"/>
-Sat, 11 Jun 2016 20:17:20 -0700
-Sat, 11 Jun 2016 20:17:20 -0700
+Sat, 11 Jun 2016 20:32:40 -0700
+Sat, 11 Jun 2016 20:32:40 -0700
 Jekyll v2.5.3
 
   
@@ -59,12 +59,15 @@
 pApache Kylin mainly use HBase to storage cube data. The performance 
of HBase cluster impacts on the query performance of Kylin directly. In common 
scenario, HBase is deployed with MR/Hive on one HDFS cluster, which makes that 
the resouces HBase used is limited, and the MR job affects the performance of 
HBase. These p

[2/2] kylin git commit: KYLIN-1777 Streaming cube build shouldn't check working segment

2016-06-11 Thread shaofengshi

KYLIN-1777 Streaming cube build shouldn't check working segment

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b376e242
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b376e242
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b376e242

Branch: refs/heads/master
Commit: b376e242d52757aa04ab2ed7b0b9777311699139
Parents: 0ddd907
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sun Jun 12 12:40:45 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sun Jun 12 12:40:45 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java| 17 -
 .../streaming/cube/StreamingCubeBuilder.java   |  2 +-
 2 files changed, 13 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b376e242/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 74deac4..8370611 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -397,7 +397,13 @@ public class CubeManager implements IRealizationProvider {
 }
 
 public CubeSegment appendSegment(CubeInstance cube, long startDate, long 
endDate, long startOffset, long endOffset) throws IOException {
-checkNoBuildingSegment(cube);
+return appendSegment(cube, startDate, endDate, startOffset, endOffset, 
true, true);
+}
+
+public CubeSegment appendSegment(CubeInstance cube, long startDate, long 
endDate, long startOffset, long endOffset, boolean strictChecking, boolean 
saveChange) throws IOException {
+
+if(strictChecking)
+checkNoBuildingSegment(cube);
 
 if 
(cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
 // try figure out a reasonable start if missing
@@ -418,10 +424,11 @@ public class CubeManager implements IRealizationProvider {
 CubeSegment newSegment = newSegment(cube, startDate, endDate, 
startOffset, endOffset);
 validateNewSegments(cube, newSegment);
 
-CubeUpdate cubeBuilder = new CubeUpdate(cube);
-cubeBuilder.setToAddSegs(newSegment);
-updateCube(cubeBuilder);
-
+if (saveChange) {
+CubeUpdate cubeBuilder = new CubeUpdate(cube);
+cubeBuilder.setToAddSegs(newSegment);
+updateCube(cubeBuilder);
+}
 return newSegment;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b376e242/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
--
diff --git 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index a08f13c..b4790e6 100644
--- 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -115,7 +115,7 @@ public class StreamingCubeBuilder implements 
StreamingBatchBuilder {
 CubeManager cubeManager = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
 final CubeInstance cubeInstance = 
cubeManager.reloadCubeLocal(cubeName);
 try {
-CubeSegment segment = cubeManager.appendSegment(cubeInstance, 
streamingBatch.getTimeRange().getFirst(), 
streamingBatch.getTimeRange().getSecond(), 0, 0);
+CubeSegment segment = cubeManager.appendSegment(cubeInstance, 
streamingBatch.getTimeRange().getFirst(), 
streamingBatch.getTimeRange().getSecond(), 0, 0, false, false);
 segment.setLastBuildJobID(segment.getUuid()); // give a fake job id
 segment.setInputRecords(streamingBatch.getMessages().size());
 segment.setLastBuildTime(System.currentTimeMillis());

[1/2] kylin git commit: KYLIN-1780 Potential issue in SnapshotTable.equals()

2016-06-11 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 536f94c0c -> b376e242d


KYLIN-1780 Potential issue in SnapshotTable.equals()

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0ddd907b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0ddd907b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0ddd907b

Branch: refs/heads/master
Commit: 0ddd907b127907b08179d329ad7267f4a446092f
Parents: 536f94c
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sun Jun 12 12:32:14 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sun Jun 12 12:32:14 2016 +0800

--
 .../main/java/org/apache/kylin/dict/lookup/SnapshotTable.java| 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0ddd907b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index f69e29f..b2306b2 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -170,6 +170,9 @@ public class SnapshotTable extends RootPersistentEntity 
implements ReadableTable
 return false;
 SnapshotTable that = (SnapshotTable) o;
 
+if (this.dict.equals(that.dict) == false)
+return false;
+
 //compare row by row
 if (this.rowIndices.size() != that.rowIndices.size())
 return false;
@@ -177,6 +180,7 @@ public class SnapshotTable extends RootPersistentEntity 
implements ReadableTable
 if (!ArrayUtils.isEquals(this.rowIndices.get(i), 
that.rowIndices.get(i)))
 return false;
 }
+
 return true;
 }

kylin git commit: small refactor in StreamingParser

2016-06-14 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 0a3541254 -> 5b4cbc4b6


small refactor in StreamingParser

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5b4cbc4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5b4cbc4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5b4cbc4b

Branch: refs/heads/master
Commit: 5b4cbc4b61e444093b88fdb4ba3d3afeba7468d3
Parents: 0a35412
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jun 15 09:52:17 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jun 15 09:52:47 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeSegment.java | 18 ++--
 .../kylin/source/kafka/KafkaStreamingInput.java | 18 +++-
 .../kylin/source/kafka/StreamingParser.java | 93 +---
 .../source/kafka/StringStreamingParser.java |  3 +-
 .../source/kafka/TimedJsonStreamParser.java | 57 
 5 files changed, 112 insertions(+), 77 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5b4cbc4b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index cbcee14..f79e06d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -19,13 +19,10 @@
 package org.apache.kylin.cube;
 
 import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.Dictionary;
@@ -101,6 +98,10 @@ public class CubeSegment implements 
Comparable, IRealizationSegment
 @JsonProperty("rowkey_stats")
 private List<Object[]> rowkeyStats = Lists.newArrayList();
 
+@JsonProperty("additionalInfo")
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+private HashMap<String, String> additionalInfo = new LinkedHashMap<String, 
String>();
+
 private volatile Map<Long, Short> cuboidBaseShards = 
Maps.newHashMap();//cuboid id ==> base(starting) shard for this cuboid
 
 public CubeDesc getCubeDesc() {
@@ -523,4 +524,11 @@ public class CubeSegment implements 
Comparable, IRealizationSegment
 this.indexPath = indexPath;
 }
 
+public HashMap<String, String> getAdditionalInfo() {
+return additionalInfo;
+}
+
+public void setAdditionalInfo(HashMap<String, String> additionalInfo) {
+this.additionalInfo = additionalInfo;
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5b4cbc4b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
index c05119f..564c221 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
@@ -40,6 +40,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
+import com.google.common.base.Function;
 import kafka.cluster.Broker;
 import kafka.javaapi.FetchResponse;
 import kafka.javaapi.PartitionMetadata;
@@ -49,11 +50,14 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
 import org.apache.kylin.engine.streaming.IStreamingInput;
 import org.apache.kylin.common.util.StreamingBatch;
 import org.apache.kylin.common.util.StreamingMessage;
 import org.apache.kylin.engine.streaming.StreamingConfig;
 import org.apache.kylin.engine.streaming.StreamingManager;
+import org.apache.kylin.metadata.model.IntermediateColumnDesc;
+import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.source.kafka.config.KafkaClusterConfig;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
@@ -64,6 +68,8 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 
+import javax.annotation.

kylin git commit: create row_count folder with the local account

2016-06-15 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 5fee54742 -> 8c05e078d


create row_count folder with the local account

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8c05e078
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8c05e078
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8c05e078

Branch: refs/heads/master
Commit: 8c05e078dd69125a0e7fac8eb7960ee3456e9c26
Parents: 5fee547
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jun 15 16:01:24 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jun 15 16:01:24 2016 +0800

--
 core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8c05e078/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 24032fa..d064cb1 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -146,6 +146,7 @@ public class JoinedFlatTable {
 final Map<String, String> tableAliasMap = 
buildTableAliasMap(intermediateTableDesc.getDataModel());
 final StringBuilder sql = new StringBuilder();
 final String factTbl = 
intermediateTableDesc.getDataModel().getFactTable();
+sql.append("dfs -mkdir -p " + outputDir + ";");
 sql.append("INSERT OVERWRITE DIRECTORY '" + outputDir + "' SELECT 
count(*) from " + factTbl + " " + tableAliasMap.get(factTbl) + "\n");
 appendWhereStatement(intermediateTableDesc, sql, tableAliasMap);
 return sql.toString();

kylin git commit: remove redundant code

2016-06-15 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master c965250b4 -> cdcde050d


remove redundant code

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cdcde050
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cdcde050
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cdcde050

Branch: refs/heads/master
Commit: cdcde050d18ae72408f91cf2d3dc894a5798d319
Parents: c965250
Author: shaofengshi <shaofeng...@apache.org>
Authored: Wed Jun 15 16:46:37 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Wed Jun 15 16:47:02 2016 +0800

--
 .../main/java/org/apache/kylin/source/hive/HiveMRInput.java   | 7 +--
 1 file changed, 1 insertion(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cdcde050/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 76dda34..a444903 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -112,12 +112,7 @@ public class HiveMRInput implements IMRInput {
 final String cubeName = 
CubingExecutableUtil.getCubeName(jobFlow.getParams());
 
 final String rowCountOutputDir = 
JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId()) + "/row_count";
-try {
-FileSystem fs = 
FileSystem.get(HadoopUtil.getCurrentConfiguration());
-fs.mkdirs(new Path(rowCountOutputDir));
-} catch (IOException e) {
-throw new RuntimeException("Failed to create HDFS dir " + 
rowCountOutputDir, e);
-}
+
 jobFlow.addTask(createCountHiveTableStep(conf, flatHiveTableDesc, 
jobFlow.getId(), rowCountOutputDir));
 jobFlow.addTask(createFlatHiveTableStep(conf, flatHiveTableDesc, 
jobFlow.getId(), cubeName, rowCountOutputDir));
 AbstractExecutable task = 
createLookupHiveViewMaterializationStep(jobFlow.getId());

kylin git commit: KYLIN-1780 Potential issue in SnapshotTable.equals()

2016-06-13 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.4-rc 871154447 -> e3cbbcf13


KYLIN-1780 Potential issue in SnapshotTable.equals()

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e3cbbcf1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e3cbbcf1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e3cbbcf1

Branch: refs/heads/1.4-rc
Commit: e3cbbcf13faa811b352b336c1835105abfcf60d8
Parents: 8711544
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sun Jun 12 12:32:14 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Tue Jun 14 09:29:44 2016 +0800

--
 .../main/java/org/apache/kylin/dict/lookup/SnapshotTable.java| 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e3cbbcf1/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index b179307..b8b1918 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -159,6 +159,9 @@ public class SnapshotTable extends RootPersistentEntity 
implements ReadableTable
 return false;
 SnapshotTable that = (SnapshotTable) o;
 
+if (this.dict.equals(that.dict) == false)
+return false;
+
 //compare row by row
 if (this.rowIndices.size() != that.rowIndices.size())
 return false;
@@ -166,6 +169,7 @@ public class SnapshotTable extends RootPersistentEntity 
implements ReadableTable
 if (!ArrayUtils.isEquals(this.rowIndices.get(i), 
that.rowIndices.get(i)))
 return false;
 }
+
 return true;
 }

kylin git commit: KYLIN-1758 should set hive conf when creating materialized tables

2016-06-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 6f6dcf00c -> 4018e6671


KYLIN-1758 should set hive conf when creating materialized tables


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4018e667
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4018e667
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4018e667

Branch: refs/heads/master
Commit: 4018e6671f6f4e23f4493286881fa47f302c6372
Parents: 6f6dcf0
Author: shaofengshi <shaofeng...@apache.org>
Authored: Sat Jun 4 20:58:30 2016 +0800
Committer: shaofengshi <shaofeng...@apache.org>
Committed: Sat Jun 4 20:58:30 2016 +0800

--
 .../org/apache/kylin/job/JoinedFlatTable.java   | 25 +---
 .../kylin/provision/BuildIIWithStream.java  |  7 +-
 .../apache/kylin/source/hive/HiveMRInput.java   | 20 
 3 files changed, 16 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4018e667/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index d625ad7..24032fa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -55,15 +55,16 @@ public class JoinedFlatTable {
 return storageDfsDir + "/" + intermediateTableDesc.getTableName();
 }
 
-public static String generateHiveSetStatements(JobEngineConfig 
engineConfig) throws IOException {
+public static String generateHiveSetStatements(JobEngineConfig 
engineConfig) {
 StringBuilder buffer = new StringBuilder();
-File hadoopPropertiesFile = new 
File(engineConfig.getHiveConfFilePath());
 
-if (hadoopPropertiesFile.exists()) {
-DocumentBuilderFactory factory = 
DocumentBuilderFactory.newInstance();
-DocumentBuilder builder;
-Document doc;
-try {
+try {
+File hadoopPropertiesFile = new 
File(engineConfig.getHiveConfFilePath());
+
+if (hadoopPropertiesFile.exists()) {
+DocumentBuilderFactory factory = 
DocumentBuilderFactory.newInstance();
+DocumentBuilder builder;
+Document doc;
 builder = factory.newDocumentBuilder();
 doc = builder.parse(hadoopPropertiesFile);
 NodeList nl = doc.getElementsByTagName("property");
@@ -74,14 +75,10 @@ public class JoinedFlatTable {
 buffer.append("SET " + name + "=" + value + ";\n");
 }
 }
-
-} catch (ParserConfigurationException e) {
-throw new IOException(e);
-} catch (SAXException e) {
-throw new IOException(e);
 }
+} catch (Exception e) {
+throw new RuntimeException("Failed to parse hive conf file ", e);
 }
-
 return buffer.toString();
 }
 
@@ -114,7 +111,7 @@ public class JoinedFlatTable {
 return ddl.toString();
 }
 
-public static String generateInsertDataStatement(IJoinedFlatTableDesc 
intermediateTableDesc, JobEngineConfig engineConfig) throws IOException {
+public static String generateInsertDataStatement(IJoinedFlatTableDesc 
intermediateTableDesc, JobEngineConfig engineConfig) {
 StringBuilder sql = new StringBuilder();
 sql.append(generateHiveSetStatements(engineConfig));
 sql.append("INSERT OVERWRITE TABLE " + 
intermediateTableDesc.getTableName() + " " + 
generateSelectDataStatement(intermediateTableDesc) + ";").append("\n");

http://git-wip-us.apache.org/repos/asf/kylin/blob/4018e667/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
index c0f6f63..1a335c6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
@@ -148,12 +148,7 @@ public class BuildIIWithStream {
 final String dropTableHql = 
JoinedFlatTable.generateDropTableStatement(intermediateTableDesc);
 final String createTableHql = 
JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, 
JobBuilderSupport.getJobWorkingDir(jobEngineConfig, uuid));
 String insertDataHq

1 2 3 4 5 6 7 8 9 10 >

1 - 100 of 3417 matches

Mail list logo