[1/2] kylin git commit: KYLIN-2171 fix RAT reported issues

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master eea4271b3 -> 0aca5d5b7


KYLIN-2171 fix RAT reported issues


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6af36a5a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6af36a5a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6af36a5a

Branch: refs/heads/master
Commit: 6af36a5ae8fbcb82ffe45d3e7a21d0c030084051
Parents: eea4271
Author: shaofengshi 
Authored: Thu Nov 10 11:22:55 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 10 14:42:34 2016 +0800

--
 .../kylin/measure/AggregatorMemEstimateTest.java  | 18 ++
 dev-support/sync_hbase_cdh_branches.sh| 17 +
 2 files changed, 35 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6af36a5a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
 
b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 2883923..3adec73 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.kylin.measure;
 
 import com.google.common.collect.Lists;

http://git-wip-us.apache.org/repos/asf/kylin/blob/6af36a5a/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh 
b/dev-support/sync_hbase_cdh_branches.sh
index f52284a..472f402 100644
--- a/dev-support/sync_hbase_cdh_branches.sh
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -1,5 +1,22 @@
 #!/bin/bash
 
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # 
 
 base=master

[2/2] kylin git commit: KYLIN-2171 update version to 1.6.1-SNAPSHOT

[shaofengshi]

KYLIN-2171 update version to 1.6.1-SNAPSHOT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0aca5d5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0aca5d5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0aca5d5b

Branch: refs/heads/master
Commit: 0aca5d5b7921833f9fb97b8342166868f65b681b
Parents: 6af36a5
Author: shaofengshi 
Authored: Thu Nov 10 14:47:36 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 10 14:47:36 2016 +0800

--
 assembly/pom.xml   | 2 +-
 atopcalcite/pom.xml| 2 +-
 core-common/pom.xml| 2 +-
 .../src/main/java/org/apache/kylin/common/KylinVersion.java| 2 +-
 core-cube/pom.xml  | 2 +-
 core-dictionary/pom.xml| 2 +-
 core-job/pom.xml   | 2 +-
 core-metadata/pom.xml  | 2 +-
 core-storage/pom.xml   | 2 +-
 engine-mr/pom.xml  | 2 +-
 engine-spark/pom.xml   | 2 +-
 jdbc/pom.xml   | 2 +-
 kylin-it/pom.xml   | 2 +-
 pom.xml| 2 +-
 query/pom.xml  | 2 +-
 server-base/pom.xml| 2 +-
 server/pom.xml | 2 +-
 source-hive/pom.xml| 2 +-
 source-kafka/pom.xml   | 2 +-
 storage-hbase/pom.xml  | 2 +-
 tomcat-ext/pom.xml | 2 +-
 tool/pom.xml   | 2 +-
 22 files changed, 22 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0aca5d5b/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index e6f83a8..73f9e12 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
 
 kylin
 org.apache.kylin
-1.6.0-SNAPSHOT
+1.6.1-SNAPSHOT
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/0aca5d5b/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 177b933..c1dba66 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0aca5d5b/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 780fd82..1bc5836 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0aca5d5b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 4a55ae8..0e4147a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -69,7 +69,7 @@ public class KylinVersion {
 /**
  * Require MANUAL updating kylin version per ANY upgrading.
  */
-private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("1.6.0");
+private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("1.6.1");
 
 private static final Set SIGNATURE_INCOMPATIBLE_REVISIONS = 
new HashSet();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0aca5d5b/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 39bba59..b013493 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.1-SNAPSHOT
 
 
 

svn commit: r16937 - /dev/kylin/apache-kylin-1.6.0-rc1/

[shaofengshi]

Author: shaofengshi
Date: Thu Nov 10 06:22:29 2016
New Revision: 16937

Log:
delete old bin packages

Removed:
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc

dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.md5

svn commit: r16936 - /dev/kylin/apache-kylin-1.6.0-rc1/

[shaofengshi]

Author: shaofengshi
Date: Thu Nov 10 06:21:43 2016
New Revision: 16936

Log:
add 1.6.0 src

Added:
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz   (with 
props)
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc.md5
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc.sha1
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.md5
dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.sha1

Added: dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz
==
Binary file - no diff available.

Propchange: dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz
--
svn:mime-type = application/octet-stream

Added: dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc
==
--- dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc (added)
+++ dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc Thu Nov 
10 06:21:43 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJYJAobAAoJEKvJ+IVvz0cghfwQAJjT40SyZkAsvgCxP1NuFQtI
+j+6Wo34jZHKtAbNH7OBfN7zzoesF5mJkJo8apiDI3Jk2aVEb6zj5oi6IxC7447hL
+pkbXyX/64xplh+NoOYO29ZqJxsEP5ir7LpxSNnveYjZxCwixOz1oEIN8BG33P0an
+PCeGqmuYtiR47gfgjDxxdo8musQ2k2rgx5WS7ibzrwF43bIRUjRLE2dil6Gwntrg
+r53llhXDDUcVsJrLtrZjB1M8/zHdqxCQyl2aDWgeoImaYFH/G4AC6F8ypHvOgkJL
+G2/BQeM9OjTFFDBzEdIQEvPuYcf75xKaVwRcNtGQBMopdBZX51EnN+Zq4vUhknwc
+YIUpw4zZxoKN36Wy6yOUtkXlyxm7bEw6i540yZqchFmz5YaDbHxbhwXjBfu8289N
+NlF7fWmyB7+2oiOO2+MHQtE1VsiRWHQEISq5bC4XHY58UsyFeXH2noeDWfryp9p+
+o3odg4j2mxIlHukbxPY/37vxUSHipTgmgc1fFIGN76ta/sgPX5jBZNUStfJftKDI
+MEpBRe4+Dg3gXn7WC+u9ow7HU2WugoulxN4qD4qDk9TvTsSnbmG1mvKn3oq9Dwhr
+WHtHtz8nLx6jTyw6c75u3/bTMuZSVShiw1Zq8PsjIIQJ45xanPLBxhNfumHP5tRj
+PgEk4vIoIT2k5iV7xKwQ
+=8cO7
+-END PGP SIGNATURE-

Added: dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc.md5
==
--- dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc.md5 
(added)
+++ dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc.md5 Thu 
Nov 10 06:21:43 2016
@@ -0,0 +1 @@
+704d773f3b79fa75e00b75a00c62bef2
\ No newline at end of file

Added: dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc.sha1
==
--- dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc.sha1 
(added)
+++ dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.asc.sha1 Thu 
Nov 10 06:21:43 2016
@@ -0,0 +1 @@
+886f6ff90b44b22ecad6c0fbdb61d52fb890b0c1
\ No newline at end of file

Added: dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.md5
==
--- dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.md5 (added)
+++ dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.md5 Thu Nov 
10 06:21:43 2016
@@ -0,0 +1 @@
+68bd217f987d259ecf29dc192f2edb02
\ No newline at end of file

Added: dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.sha1
==
--- dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.sha1 (added)
+++ dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-src.tar.gz.sha1 Thu Nov 
10 06:21:43 2016
@@ -0,0 +1 @@
+504c8c57936be168e341732fd5d548d3f3bfe314
\ No newline at end of file

kylin git commit: [maven-release-plugin] prepare for next development iteration

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 30a45dcfe -> b5079074f


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b5079074
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b5079074
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b5079074

Branch: refs/heads/v1.6.0-rc1
Commit: b5079074f5ce87cb8ad18b2e047fd770fc945724
Parents: 30a45dc
Author: shaofengshi 
Authored: Thu Nov 10 05:52:39 2016 +
Committer: shaofengshi 
Committed: Thu Nov 10 05:52:39 2016 +

--
 assembly/pom.xml| 2 +-
 atopcalcite/pom.xml | 2 +-
 core-common/pom.xml | 2 +-
 core-cube/pom.xml   | 2 +-
 core-dictionary/pom.xml | 2 +-
 core-job/pom.xml| 2 +-
 core-metadata/pom.xml   | 2 +-
 core-storage/pom.xml| 2 +-
 engine-mr/pom.xml   | 2 +-
 engine-spark/pom.xml| 2 +-
 jdbc/pom.xml| 2 +-
 kylin-it/pom.xml| 2 +-
 pom.xml | 4 ++--
 query/pom.xml   | 2 +-
 server-base/pom.xml | 2 +-
 server/pom.xml  | 2 +-
 source-hive/pom.xml | 2 +-
 source-kafka/pom.xml| 2 +-
 storage-hbase/pom.xml   | 2 +-
 tomcat-ext/pom.xml  | 2 +-
 tool/pom.xml| 2 +-
 21 files changed, 22 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b5079074/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 17716bc..1489728 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.6.0
+1.6.1-SNAPSHOT
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5079074/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 5185962..bd434f7 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5079074/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index a2ca10f..028be8f 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5079074/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 7cfcf9f..2602c41 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5079074/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index d712272..1691aff 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5079074/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 04642a8..65a72ca 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5079074/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index b0193eb..7a08944 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5079074/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index ede1dfd..85c4c5d 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5079074/engine-mr/pom.xml
--
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 47e7301..09aad8c 

[kylin] Git Push Summary

[shaofengshi]

Repository: kylin
Updated Tags:  refs/tags/kylin-1.6.0 [created] fd524cfb4

kylin git commit: [maven-release-plugin] prepare release kylin-1.6.0

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 bdc1f0718 -> 30a45dcfe


[maven-release-plugin] prepare release kylin-1.6.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30a45dcf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30a45dcf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30a45dcf

Branch: refs/heads/v1.6.0-rc1
Commit: 30a45dcfed0ea84328787ceab22e843ad560c352
Parents: bdc1f07
Author: shaofengshi 
Authored: Thu Nov 10 05:51:14 2016 +
Committer: shaofengshi 
Committed: Thu Nov 10 05:51:14 2016 +

--
 assembly/pom.xml|  5 ++---
 atopcalcite/pom.xml |  5 ++---
 core-common/pom.xml |  5 ++---
 core-cube/pom.xml   |  5 ++---
 core-dictionary/pom.xml |  5 ++---
 core-job/pom.xml|  5 ++---
 core-metadata/pom.xml   |  5 ++---
 core-storage/pom.xml|  5 ++---
 engine-mr/pom.xml   |  5 ++---
 engine-spark/pom.xml|  5 ++---
 jdbc/pom.xml|  5 ++---
 kylin-it/pom.xml| 13 ++---
 pom.xml | 11 +--
 query/pom.xml   |  5 ++---
 server-base/pom.xml |  5 ++---
 server/pom.xml  |  5 ++---
 source-hive/pom.xml |  5 ++---
 source-kafka/pom.xml|  5 ++---
 storage-hbase/pom.xml   |  8 +++-
 tomcat-ext/pom.xml  |  6 ++
 tool/pom.xml|  5 ++---
 21 files changed, 50 insertions(+), 73 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/30a45dcf/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index e6f83a8..17716bc 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -17,12 +17,11 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
 kylin
 org.apache.kylin
-1.6.0-SNAPSHOT
+1.6.0
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/30a45dcf/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 177b933..5185962 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 atopcalcite
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.0
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/30a45dcf/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 780fd82..a2ca10f 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 kylin-core-common
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.0
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/30a45dcf/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 39bba59..7cfcf9f 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
  

[kylin] Git Push Summary [forced push!] [Forced Update!]

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 f9b1618b3 -> bdc1f0718 (forced update)

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 dbd154eee -> 3909d8d6f (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b95baee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b95baee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 6663adeec -> 7b95baee1 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b95baee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b95baee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-  

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b95baee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b95baee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b95baee

Branch: refs/heads/yang21-hbase1.x
Commit: 7b95baee17e7fea1d85e5cf8741274f12985bd12
Parents: 1e24228
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 10 13:28:49 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b95baee/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-   

kylin git commit: [maven-release-plugin] prepare for next development iteration

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 bdc1f0718 -> f9b1618b3


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f9b1618b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f9b1618b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f9b1618b

Branch: refs/heads/v1.6.0-rc1
Commit: f9b1618b3f3bc67d6d90224cb8085c44bc53bbde
Parents: bdc1f07
Author: shaofengshi 
Authored: Thu Nov 10 03:36:49 2016 +
Committer: shaofengshi 
Committed: Thu Nov 10 03:36:49 2016 +

--
 assembly/pom.xml|  5 ++---
 atopcalcite/pom.xml |  5 ++---
 core-common/pom.xml |  5 ++---
 core-cube/pom.xml   |  5 ++---
 core-dictionary/pom.xml |  5 ++---
 core-job/pom.xml|  5 ++---
 core-metadata/pom.xml   |  5 ++---
 core-storage/pom.xml|  5 ++---
 engine-mr/pom.xml   |  5 ++---
 engine-spark/pom.xml|  5 ++---
 jdbc/pom.xml|  5 ++---
 kylin-it/pom.xml| 13 ++---
 pom.xml |  9 -
 query/pom.xml   |  5 ++---
 server-base/pom.xml |  5 ++---
 server/pom.xml  |  5 ++---
 source-hive/pom.xml |  5 ++---
 source-kafka/pom.xml|  5 ++---
 storage-hbase/pom.xml   |  8 +++-
 tomcat-ext/pom.xml  |  6 ++
 tool/pom.xml|  5 ++---
 21 files changed, 49 insertions(+), 72 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f9b1618b/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index e6f83a8..1489728 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -17,12 +17,11 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
 kylin
 org.apache.kylin
-1.6.0-SNAPSHOT
+1.6.1-SNAPSHOT
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/f9b1618b/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 177b933..bd434f7 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 atopcalcite
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f9b1618b/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 780fd82..028be8f 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 kylin-core-common
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f9b1618b/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 39bba59..2602c41 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 

kylin git commit: KYLIN-2171 fix RAT reported issues [Forced Update!]

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 61183acff -> bdc1f0718 (forced update)


KYLIN-2171 fix RAT reported issues


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bdc1f071
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bdc1f071
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bdc1f071

Branch: refs/heads/v1.6.0-rc1
Commit: bdc1f0718ad29576e941838d9e2b94116cbca558
Parents: 516749e
Author: shaofengshi 
Authored: Thu Nov 10 11:22:55 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 10 03:32:38 2016 +

--
 .../kylin/measure/AggregatorMemEstimateTest.java  | 18 ++
 dev-support/sync_hbase_cdh_branches.sh| 17 +
 2 files changed, 35 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bdc1f071/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
 
b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 2883923..3adec73 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.kylin.measure;
 
 import com.google.common.collect.Lists;

http://git-wip-us.apache.org/repos/asf/kylin/blob/bdc1f071/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh 
b/dev-support/sync_hbase_cdh_branches.sh
index f52284a..472f402 100644
--- a/dev-support/sync_hbase_cdh_branches.sh
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -1,5 +1,22 @@
 #!/bin/bash
 
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # 
 
 base=master

kylin git commit: rat

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 f43132efc -> 61183acff


rat


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61183acf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61183acf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61183acf

Branch: refs/heads/v1.6.0-rc1
Commit: 61183acff0cc436e84135f28e0525222939f880d
Parents: f43132e
Author: shaofengshi 
Authored: Thu Nov 10 11:26:38 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 10 11:26:38 2016 +0800

--
 .../kylin/measure/AggregatorMemEstimateTest.java  | 18 ++
 1 file changed, 18 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61183acf/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
 
b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 2883923..3adec73 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.kylin.measure;
 
 import com.google.common.collect.Lists;

kylin git commit: KYLIN-2171 fix RAT

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 516749e45 -> f43132efc


KYLIN-2171 fix RAT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f43132ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f43132ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f43132ef

Branch: refs/heads/v1.6.0-rc1
Commit: f43132efc0dd169161682db0703f58a0c291e066
Parents: 516749e
Author: shaofengshi 
Authored: Thu Nov 10 11:22:55 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 10 11:22:55 2016 +0800

--
 dev-support/sync_hbase_cdh_branches.sh | 17 +
 1 file changed, 17 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f43132ef/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh 
b/dev-support/sync_hbase_cdh_branches.sh
index f52284a..472f402 100644
--- a/dev-support/sync_hbase_cdh_branches.sh
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -1,5 +1,22 @@
 #!/bin/bash
 
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # 
 
 base=master

kylin git commit: KYLIN-2170 refactor doMap()/doRecuce()/doCleanup()

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 5ea095a32 -> 6b068e36e


KYLIN-2170 refactor doMap()/doRecuce()/doCleanup()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b068e36
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b068e36
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b068e36

Branch: refs/heads/master
Commit: 6b068e36e2a1fb7b18bddcea2d73ee93c387ab99
Parents: 5ea095a
Author: Li Yang 
Authored: Thu Nov 10 10:50:24 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 10 10:50:36 2016 +0800

--
 .../mr/steps/fdc2/FactDistinctColumnsCombiner2.java |  2 +-
 .../mr/steps/fdc2/FactDistinctColumnsMapperBase2.java   |  2 +-
 .../mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java   |  4 ++--
 .../kylin/source/kafka/hadoop/KafkaFlatTableMapper.java | 12 
 4 files changed, 8 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6b068e36/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java
index 6ff07f0..6652f4e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsCombiner2.java
@@ -34,7 +34,7 @@ public class FactDistinctColumnsCombiner2 extends 
KylinReducer values, 
Context context) throws IOException, InterruptedException {
+public void doReduce(SelfDefineSortableKey key, Iterable values, 
Context context) throws IOException, InterruptedException {
 
 // for hll, each key only has one output, no need to do local combine;
 // for normal col, values are empty text

http://git-wip-us.apache.org/repos/asf/kylin/blob/6b068e36/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java
index 2e9a2dc..6238d22 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctColumnsMapperBase2.java
@@ -41,7 +41,7 @@ import java.util.List;
 
 /**
  */
-public class FactDistinctColumnsMapperBase2 extends 
KylinMapper {
+abstract public class FactDistinctColumnsMapperBase2 extends 
KylinMapper {
 
 protected String cubeName;
 protected CubeInstance cube;

http://git-wip-us.apache.org/repos/asf/kylin/blob/6b068e36/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java
index dfc46b6..bdf631d 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/fdc2/FactDistinctHiveColumnsMapper2.java
@@ -131,7 +131,7 @@ public class FactDistinctHiveColumnsMapper2 extends 
FactDistinctColumnsMa
 }
 
 @Override
-public void map(KEYIN key, Object record, Context context) throws 
IOException, InterruptedException {
+public void doMap(KEYIN key, Object record, Context context) throws 
IOException, InterruptedException {
 String[] row = flatTableInputFormat.parseMapperInput(record);
 
 keyBuffer.clear();
@@ -208,7 +208,7 @@ public class FactDistinctHiveColumnsMapper2 extends 
FactDistinctColumnsMa
 }
 
 @Override
-protected void cleanup(Context context) throws IOException, 
InterruptedException {
+protected void doCleanup(Context context) throws IOException, 
InterruptedException {
 if (collectStatistics) {
 ByteBuffer hllBuf = 
ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
 // output each cuboid's hll to reducer, key is 0 - cuboidId

kylin git commit: KYLIN-1726 fix a NPE

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master 862aaabe6 -> 5ea095a32


KYLIN-1726 fix a NPE


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5ea095a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5ea095a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5ea095a3

Branch: refs/heads/master
Commit: 5ea095a3251a6588331a26d57ae3bdee4eb61fbb
Parents: 862aaab
Author: shaofengshi 
Authored: Thu Nov 10 10:48:45 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 10 10:48:45 2016 +0800

--
 .../main/java/org/apache/kylin/source/SourcePartition.java   | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5ea095a3/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java 
b/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java
index cf89580..e489704 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java
@@ -104,8 +104,12 @@ public class SourcePartition {
 copy.setEndDate(origin.getEndDate());
 copy.setStartOffset(origin.getStartOffset());
 copy.setEndOffset(origin.getEndOffset());
-copy.setSourcePartitionOffsetStart(new 
HashMap<>(origin.getSourcePartitionOffsetStart()));
-copy.setSourcePartitionOffsetEnd(new 
HashMap<>(origin.getSourcePartitionOffsetEnd()));
+if (origin.getSourcePartitionOffsetStart() != null) {
+copy.setSourcePartitionOffsetStart(new 
HashMap<>(origin.getSourcePartitionOffsetStart()));
+}
+if (origin.getSourcePartitionOffsetEnd() != null) {
+copy.setSourcePartitionOffsetEnd(new 
HashMap<>(origin.getSourcePartitionOffsetEnd()));
+}
 return copy;
 }
 }

kylin git commit: KYLIN-1726 KafkaSource may get wrong start offset when partition added. [Forced Update!]

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 c43f38f48 -> 516749e45 (forced update)


KYLIN-1726 KafkaSource may get wrong start offset when partition added.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/516749e4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/516749e4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/516749e4

Branch: refs/heads/v1.6.0-rc1
Commit: 516749e457b00b9ab1260fb1150fd436cff33a61
Parents: 5d166aa
Author: shaofengshi 
Authored: Wed Nov 9 09:28:09 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 10 10:47:31 2016 +0800

--
 .../apache/kylin/source/SourcePartition.java| 16 +++--
 .../apache/kylin/source/kafka/KafkaSource.java  | 38 
 .../kylin/source/kafka/util/KafkaClient.java|  5 +--
 3 files changed, 48 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/516749e4/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java 
b/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java
index 8ba749d..e489704 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/SourcePartition.java
@@ -18,8 +18,11 @@
 
 package org.apache.kylin.source;
 
+import java.util.HashMap;
 import java.util.Map;
 
+import com.google.common.base.Objects;
+
 /**
  */
 public class SourcePartition {
@@ -90,14 +93,23 @@ public class SourcePartition {
 this.sourcePartitionOffsetEnd = sourcePartitionOffsetEnd;
 }
 
+@Override
+public String toString() {
+return Objects.toStringHelper(this).add("startDate", 
startDate).add("endDate", endDate).add("startOffset", 
startOffset).add("endOffset", endOffset).add("sourcePartitionOffsetStart", 
sourcePartitionOffsetStart.toString()).add("sourcePartitionOffsetEnd", 
sourcePartitionOffsetEnd.toString()).toString();
+}
+
 public static SourcePartition getCopyOf(SourcePartition origin) {
 SourcePartition copy = new SourcePartition();
 copy.setStartDate(origin.getStartDate());
 copy.setEndDate(origin.getEndDate());
 copy.setStartOffset(origin.getStartOffset());
 copy.setEndOffset(origin.getEndOffset());
-
copy.setSourcePartitionOffsetStart(origin.getSourcePartitionOffsetStart());
-copy.setSourcePartitionOffsetEnd(origin.getSourcePartitionOffsetEnd());
+if (origin.getSourcePartitionOffsetStart() != null) {
+copy.setSourcePartitionOffsetStart(new 
HashMap<>(origin.getSourcePartitionOffsetStart()));
+}
+if (origin.getSourcePartitionOffsetEnd() != null) {
+copy.setSourcePartitionOffsetEnd(new 
HashMap<>(origin.getSourcePartitionOffsetEnd()));
+}
 return copy;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/516749e4/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
index 7a5d94f..b0c8e7f 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
@@ -23,6 +23,7 @@ import java.util.Map;
 
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.common.PartitionInfo;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.IMRInput;
@@ -34,12 +35,16 @@ import org.apache.kylin.source.ReadableTable;
 import org.apache.kylin.source.SourcePartition;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.apache.kylin.source.kafka.util.KafkaClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 
 //used by reflection
 public class KafkaSource implements ISource {
 
+private static final Logger logger = 
LoggerFactory.getLogger(KafkaSource.class);
+
 @SuppressWarnings("unchecked")
 @Override
 public  I adaptToBuildEngine(Class engineInterface) {
@@ -71,32 +76,50 @@ public class KafkaSource implements ISource {
 if (result.getStartOffset() == 0) {
 final CubeSegment last = cube.getLastSegment();
 if (last != null) {
+logger.debug("Last segment exists, continue from last segment 
" 

kylin git commit: KYLIN-2170 refactor doMap/doReduce/doCleanup

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 309593b40 -> 862aaabe6


KYLIN-2170 refactor doMap/doReduce/doCleanup


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/862aaabe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/862aaabe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/862aaabe

Branch: refs/heads/master
Commit: 862aaabe6990fff7bfbfedbe72b13e1ffa370ba6
Parents: 309593b
Author: Li Yang 
Authored: Wed Nov 9 16:11:24 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 9 16:59:24 2016 +0800

--
 build/conf/kylin_job_conf_inmem.xml |  2 +-
 .../org/apache/kylin/engine/mr/KylinMapper.java | 47 +-
 .../apache/kylin/engine/mr/KylinReducer.java| 51 ++-
 .../engine/mr/steps/BaseCuboidMapperBase.java   |  2 +-
 .../kylin/engine/mr/steps/CuboidReducer.java|  2 +-
 .../mr/steps/FactDistinctColumnPartitioner.java |  1 +
 .../mr/steps/FactDistinctColumnsCombiner.java   |  2 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java |  2 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 42 ++--
 .../mr/steps/FactDistinctHiveColumnsMapper.java | 38 +--
 .../engine/mr/steps/HiveToBaseCuboidMapper.java |  2 +-
 .../engine/mr/steps/InMemCuboidMapper.java  | 28 
 .../engine/mr/steps/InMemCuboidReducer.java |  2 +-
 .../engine/mr/steps/MergeCuboidMapper.java  |  2 +-
 .../kylin/engine/mr/steps/NDCuboidMapper.java   |  2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  | 16 ++---
 .../steps/RowKeyDistributionCheckerReducer.java |  2 +-
 .../cardinality/ColumnCardinalityMapper.java| 26 
 .../cardinality/ColumnCardinalityReducer.java   | 40 +---
 .../storage/hbase/steps/CubeHFileMapper.java|  2 +-
 .../hbase/steps/RangeKeyDistributionMapper.java | 14 ++--
 .../steps/RangeKeyDistributionReducer.java  | 68 +---
 .../storage/hbase/steps/RowValueDecoder.java|  4 --
 23 files changed, 227 insertions(+), 170 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/862aaabe/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index d363507..f229afc 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -94,7 +94,7 @@
 
 
 mapreduce.map.java.opts
--Xmx2700m
+-Xmx2700m -XX:OnOutOfMemoryError="kill -9 %p"
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/862aaabe/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index 29c6844..a527b3d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.engine.mr;
 
+import java.io.IOException;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.slf4j.Logger;
@@ -25,11 +27,54 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-public class KylinMapper extends 
Mapper {
+abstract public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
 protected void bindCurrentConfiguration(Configuration conf) {
 logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 HadoopUtil.setCurrentConfiguration(conf);
 }
+
+@Override
+final public void map(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, InterruptedException {
+try {
+doMap(key, value, context);
+} catch (IOException ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+} catch (InterruptedException ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+} catch (RuntimeException ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+} catch (Error ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+}
+}
+
+abstract protected void doMap(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, 
InterruptedException;
+
+@Override
+final protected void 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/6663adee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6663adee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dbd154ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dbd154ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dbd154ee

Branch: refs/heads/yang21-cdh5.7
Commit: dbd154eee9a6249d36c2c400b07ab508ef67f8ea
Parents: 6663ade
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Li Yang 
Committed: Wed Nov 9 08:42:28 2016 +

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dbd154ee/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/dbd154ee/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/dbd154ee/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/dbd154ee/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6663adee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6663adee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6663adee

Branch: refs/heads/yang21-cdh5.7
Commit: 6663adeec88696496b3d459a3db97cebca59b6d0
Parents: 1e24228
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 9 08:40:27 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6663adee/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[1/4] kylin git commit: KYLIN-2170 refactor doMap/doReduce/doCleanup [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 6d6486f30 -> dbd154eee (forced update)


KYLIN-2170 refactor doMap/doReduce/doCleanup


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e24228e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e24228e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e24228e

Branch: refs/heads/yang21-cdh5.7
Commit: 1e24228ea9e4a070923a854e6a50fbe561ccc378
Parents: 3b07c26
Author: Li Yang 
Authored: Wed Nov 9 16:11:24 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 9 16:11:24 2016 +0800

--
 build/conf/kylin_job_conf_inmem.xml |  2 +-
 .../org/apache/kylin/engine/mr/KylinMapper.java | 47 +-
 .../apache/kylin/engine/mr/KylinReducer.java| 51 ++-
 .../engine/mr/steps/BaseCuboidMapperBase.java   |  2 +-
 .../kylin/engine/mr/steps/CuboidReducer.java|  2 +-
 .../mr/steps/FactDistinctColumnPartitioner.java |  1 +
 .../mr/steps/FactDistinctColumnsCombiner.java   |  2 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java |  2 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 42 ++--
 .../mr/steps/FactDistinctHiveColumnsMapper.java | 38 +--
 .../engine/mr/steps/HiveToBaseCuboidMapper.java |  2 +-
 .../engine/mr/steps/InMemCuboidMapper.java  | 28 
 .../engine/mr/steps/InMemCuboidReducer.java |  2 +-
 .../engine/mr/steps/MergeCuboidMapper.java  |  2 +-
 .../kylin/engine/mr/steps/NDCuboidMapper.java   |  2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  | 16 ++---
 .../steps/RowKeyDistributionCheckerReducer.java |  2 +-
 .../cardinality/ColumnCardinalityMapper.java| 26 
 .../cardinality/ColumnCardinalityReducer.java   | 40 +---
 .../storage/hbase/steps/CubeHFileMapper.java|  2 +-
 .../hbase/steps/RangeKeyDistributionMapper.java | 14 ++--
 .../steps/RangeKeyDistributionReducer.java  | 68 +---
 .../storage/hbase/steps/RowValueDecoder.java|  4 --
 23 files changed, 227 insertions(+), 170 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e24228e/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index d6799d5..7e6dc08 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -94,7 +94,7 @@
 
 
 mapreduce.map.java.opts
--Xmx2700m
+-Xmx2700m -XX:OnOutOfMemoryError="kill -9 %p"
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e24228e/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index 29c6844..a527b3d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.engine.mr;
 
+import java.io.IOException;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.slf4j.Logger;
@@ -25,11 +27,54 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-public class KylinMapper extends 
Mapper {
+abstract public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
 protected void bindCurrentConfiguration(Configuration conf) {
 logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 HadoopUtil.setCurrentConfiguration(conf);
 }
+
+@Override
+final public void map(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, InterruptedException {
+try {
+doMap(key, value, context);
+} catch (IOException ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+} catch (InterruptedException ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+} catch (RuntimeException ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+} catch (Error ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+}
+}
+
+abstract protected void doMap(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, 
InterruptedException;
+
+

[1/3] kylin git commit: KYLIN-2170 refactor doMap/doReduce/doCleanup [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 0d3af7e00 -> 6663adeec (forced update)


KYLIN-2170 refactor doMap/doReduce/doCleanup


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e24228e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e24228e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e24228e

Branch: refs/heads/yang21-hbase1.x
Commit: 1e24228ea9e4a070923a854e6a50fbe561ccc378
Parents: 3b07c26
Author: Li Yang 
Authored: Wed Nov 9 16:11:24 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 9 16:11:24 2016 +0800

--
 build/conf/kylin_job_conf_inmem.xml |  2 +-
 .../org/apache/kylin/engine/mr/KylinMapper.java | 47 +-
 .../apache/kylin/engine/mr/KylinReducer.java| 51 ++-
 .../engine/mr/steps/BaseCuboidMapperBase.java   |  2 +-
 .../kylin/engine/mr/steps/CuboidReducer.java|  2 +-
 .../mr/steps/FactDistinctColumnPartitioner.java |  1 +
 .../mr/steps/FactDistinctColumnsCombiner.java   |  2 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java |  2 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 42 ++--
 .../mr/steps/FactDistinctHiveColumnsMapper.java | 38 +--
 .../engine/mr/steps/HiveToBaseCuboidMapper.java |  2 +-
 .../engine/mr/steps/InMemCuboidMapper.java  | 28 
 .../engine/mr/steps/InMemCuboidReducer.java |  2 +-
 .../engine/mr/steps/MergeCuboidMapper.java  |  2 +-
 .../kylin/engine/mr/steps/NDCuboidMapper.java   |  2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  | 16 ++---
 .../steps/RowKeyDistributionCheckerReducer.java |  2 +-
 .../cardinality/ColumnCardinalityMapper.java| 26 
 .../cardinality/ColumnCardinalityReducer.java   | 40 +---
 .../storage/hbase/steps/CubeHFileMapper.java|  2 +-
 .../hbase/steps/RangeKeyDistributionMapper.java | 14 ++--
 .../steps/RangeKeyDistributionReducer.java  | 68 +---
 .../storage/hbase/steps/RowValueDecoder.java|  4 --
 23 files changed, 227 insertions(+), 170 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e24228e/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index d6799d5..7e6dc08 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -94,7 +94,7 @@
 
 
 mapreduce.map.java.opts
--Xmx2700m
+-Xmx2700m -XX:OnOutOfMemoryError="kill -9 %p"
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e24228e/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index 29c6844..a527b3d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.engine.mr;
 
+import java.io.IOException;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.slf4j.Logger;
@@ -25,11 +27,54 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-public class KylinMapper extends 
Mapper {
+abstract public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
 protected void bindCurrentConfiguration(Configuration conf) {
 logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 HadoopUtil.setCurrentConfiguration(conf);
 }
+
+@Override
+final public void map(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, InterruptedException {
+try {
+doMap(key, value, context);
+} catch (IOException ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+} catch (InterruptedException ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+} catch (RuntimeException ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+} catch (Error ex) { // KYLIN-2170
+logger.error("", ex);
+throw ex;
+}
+}
+
+abstract protected void doMap(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, 
InterruptedException;
+
+

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/6663adee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6663adee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6663adee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6663adee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6663adee

Branch: refs/heads/yang21-hbase1.x
Commit: 6663adeec88696496b3d459a3db97cebca59b6d0
Parents: 1e24228
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 9 08:40:27 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6663adee/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-  

kylin git commit: KYLIN-1826 support mutiple hive env by extending KylinConfig and HiveClient

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1826-2 [created] 3ac6490b3


KYLIN-1826 support mutiple hive env by extending KylinConfig and HiveClient


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ac6490b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ac6490b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ac6490b

Branch: refs/heads/KYLIN-1826-2
Commit: 3ac6490b36e897c6ad853f0d1eba3f9c97c88382
Parents: 309593b
Author: shaofengshi 
Authored: Wed Nov 9 16:23:19 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 9 16:23:19 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  8 ++
 .../kylin/metadata/project/ProjectInstance.java | 23 -
 .../apache/kylin/source/hive/CLIHiveClient.java | 35 +++
 .../source/hive/CreateFlatHiveTableStep.java|  2 +-
 .../kylin/source/hive/HiveClientFactory.java| 14 ++-
 .../kylin/source/hive/HiveCmdBuilder.java   | 17 +++-
 .../apache/kylin/source/hive/HiveMRInput.java   | 47 --
 .../source/hive/HiveSourceTableLoader.java  |  2 +-
 .../apache/kylin/source/hive/HiveTableMeta.java | 96 
 9 files changed, 227 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ac6490b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ee9f57c..9b3a592 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -830,4 +830,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 public void setMaxBuildingSegments(int maxBuildingSegments) {
 setProperty("kylin.cube.building.segment.max", 
String.valueOf(maxBuildingSegments));
 }
+
+public String getHiveHome() {
+return getOptional("kylin.hive.home", "");
+}
+
+public void setHiveHome(String hiveHome) {
+setProperty("kylin.hive.home", hiveHome);
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ac6490b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
index 1afc603..f9d220f 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.metadata.project;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
@@ -26,6 +27,8 @@ import java.util.TreeSet;
 import javax.annotation.Nullable;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.metadata.realization.RealizationType;
@@ -78,6 +81,11 @@ public class ProjectInstance extends RootPersistentEntity {
 @JsonProperty("ext_filters")
 private Set extFilters = new TreeSet();
 
+@JsonProperty("override_kylin_properties")
+private LinkedHashMap overrideKylinProps = new 
LinkedHashMap<>();
+
+private KylinConfigExt config;
+
 public String getResourcePath() {
 return concatResourcePath(name);
 }
@@ -285,19 +293,30 @@ public class ProjectInstance extends RootPersistentEntity 
{
 }
 }
 
+public KylinConfig getConfig() {
+return config;
+}
+
+private void setConfig(KylinConfigExt config) {
+this.config = config;
+}
+
+
 public void init() {
 if (name == null)
 name = ProjectInstance.DEFAULT_PROJECT_NAME;
 
 if (realizationEntries == null) {
-realizationEntries = new ArrayList();
+realizationEntries = new ArrayList<>();
 }
 
 if (tables == null)
-tables = new TreeSet();
+tables = new TreeSet<>();
 
 if (StringUtils.isBlank(this.name))
 throw new IllegalStateException("Project name must not be blank");
+
+this.config = KylinConfigExt.createInstance(config, 
overrideKylinProps);
 }
 
 @Override