date:20161122

svn commit: r17135 - /dev/kylin/apache-kylin-1.6.0-rc2/

2016-11-22 Thread shaofengshi

Author: shaofengshi
Date: Wed Nov 23 07:52:07 2016
New Revision: 17135

Log:
Upload release artifacts to staging

Added:
dev/kylin/apache-kylin-1.6.0-rc2/
dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz   (with 
props)
dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc
dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc.md5
dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc.sha1
dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.md5
dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.sha1

Added: dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz
==
Binary file - no diff available.

Propchange: dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz
--
svn:mime-type = application/octet-stream

Added: dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc
==
--- dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc (added)
+++ dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc Wed Nov 
23 07:52:07 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJYNUYVAAoJEKvJ+IVvz0cgWHoP/iJ5MVxibz+io5js70khu3Ur
+tZRCVt/qSaiBe/0zV9JyMTIK0oCuMYvg+7n+TKX9OtlCoj0I4psKYR3x/1plN6k3
+hUJ/OoaWiJ0htAzidC4znmMxOo/Mrnc4Y92Y8kBwzAllyiSbCVNXyxGe3lBqSVCh
+7/nS2d/q/VAsNVeuqZO/mAnhN8qrc0BaTA50hLZQFDHrMw6bKCFOMYKo3r5XI773
+RMe7xx0Sxoae4JTLOdwpqPIpDc0GNbXoLfAqp7R4cgIpZvIMweafnvwum00/78RY
+mB20yN5aUNaRL1LBn+OhPnpF8Ey8Z0kD3Lz5QzlfmioqlMduy/JNeIgMPkKTbGE0
+t7umD1FI7o/WDEXuFhQOed+W5/mriX14bPFGh3DLHtN0EPvO32CT7X1l64yI01SQ
+wWknQ2Lu7zuMH1Z2rVUV5Dzg4LN3uVjGoP01ek4faHqAJCIQsN07wjQWvTyrBGTA
+068BgtOOgdOXKGxiic8qGkJ2xTnYqa+f0sT3HXHx4VQQSwp+4R82WA+K/Z3VSHFe
+F4euyVXspd5lVAk9IjtmS48CMRl+Dr4U4EJ4/G5G5MgmStUMbXC4CaH37mzcovaB
+Kt0zsw3joXMFzNtDU2MDDZEEzE8ZqSZibT2H10EsJS+BZXBBF96iveCCxKHCAXRj
+srsLhcMv2mk2gAYwiUug
+=jqOW
+-END PGP SIGNATURE-

Added: dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc.md5
==
--- dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc.md5 
(added)
+++ dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc.md5 Wed 
Nov 23 07:52:07 2016
@@ -0,0 +1 @@
+1aea498c57f39a114d36161b9c79abd3
\ No newline at end of file

Added: dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc.sha1
==
--- dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc.sha1 
(added)
+++ dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.asc.sha1 Wed 
Nov 23 07:52:07 2016
@@ -0,0 +1 @@
+a95fd5a21b954a8b988c1ddb97ff29b514aa85ca
\ No newline at end of file

Added: dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.md5
==
--- dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.md5 (added)
+++ dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.md5 Wed Nov 
23 07:52:07 2016
@@ -0,0 +1 @@
+c275593d9a1da5869bcb6f8806de73ba
\ No newline at end of file

Added: dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.sha1
==
--- dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.sha1 (added)
+++ dev/kylin/apache-kylin-1.6.0-rc2/apache-kylin-1.6.0-src.tar.gz.sha1 Wed Nov 
23 07:52:07 2016
@@ -0,0 +1 @@
+6c5a895743609c3992e0e75ad32cb6582e6ccad7
\ No newline at end of file

[kylin] Git Push Summary

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Tags:  refs/tags/kylin-1.6.0 [created] ab1eb0c8f

kylin git commit: [maven-release-plugin] prepare release kylin-1.6.0

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 d675b7a06 -> ed6982c8b


[maven-release-plugin] prepare release kylin-1.6.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed6982c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed6982c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed6982c8

Branch: refs/heads/v1.6.0-rc2
Commit: ed6982c8b3baaad08b7e4956001339979724d9a7
Parents: d675b7a
Author: shaofengshi 
Authored: Wed Nov 23 07:35:33 2016 +
Committer: shaofengshi 
Committed: Wed Nov 23 07:35:33 2016 +

--
 assembly/pom.xml|  5 ++---
 atopcalcite/pom.xml |  5 ++---
 core-common/pom.xml |  5 ++---
 core-cube/pom.xml   |  5 ++---
 core-dictionary/pom.xml |  5 ++---
 core-job/pom.xml|  5 ++---
 core-metadata/pom.xml   |  5 ++---
 core-storage/pom.xml|  5 ++---
 engine-mr/pom.xml   |  5 ++---
 engine-spark/pom.xml|  5 ++---
 jdbc/pom.xml|  5 ++---
 kylin-it/pom.xml| 13 ++---
 pom.xml | 11 +--
 query/pom.xml   |  5 ++---
 server-base/pom.xml |  5 ++---
 server/pom.xml  |  5 ++---
 source-hive/pom.xml |  5 ++---
 source-kafka/pom.xml|  5 ++---
 storage-hbase/pom.xml   |  8 +++-
 tomcat-ext/pom.xml  |  6 ++
 tool/pom.xml|  5 ++---
 21 files changed, 50 insertions(+), 73 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ed6982c8/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index e6f83a8..17716bc 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -17,12 +17,11 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
 kylin
 org.apache.kylin
-1.6.0-SNAPSHOT
+1.6.0
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed6982c8/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 177b933..5185962 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 atopcalcite
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.0
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed6982c8/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 780fd82..a2ca10f 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 kylin-core-common
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.0
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed6982c8/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 39bba59..7cfcf9f 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0

[kylin] Git Push Summary [forced push!] [Forced Update!]

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 bf6b1eb0c -> d675b7a06 (forced update)

[kylin] Git Push Summary

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Tags:  refs/tags/kylin-1.6.0 [deleted] c65bf2e19

[kylin] Git Push Summary

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Tags:  refs/tags/kylin-1.6.0 [created] c65bf2e19

kylin git commit: [maven-release-plugin] prepare release kylin-1.6.0

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 d675b7a06 -> bf6b1eb0c


[maven-release-plugin] prepare release kylin-1.6.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bf6b1eb0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bf6b1eb0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bf6b1eb0

Branch: refs/heads/v1.6.0-rc2
Commit: bf6b1eb0cd561a60cfc5baa16c8950562209e2f2
Parents: d675b7a
Author: shaofengshi 
Authored: Wed Nov 23 06:13:57 2016 +
Committer: shaofengshi 
Committed: Wed Nov 23 06:13:57 2016 +

--
 assembly/pom.xml|  5 ++---
 atopcalcite/pom.xml |  5 ++---
 core-common/pom.xml |  5 ++---
 core-cube/pom.xml   |  5 ++---
 core-dictionary/pom.xml |  5 ++---
 core-job/pom.xml|  5 ++---
 core-metadata/pom.xml   |  5 ++---
 core-storage/pom.xml|  5 ++---
 engine-mr/pom.xml   |  5 ++---
 engine-spark/pom.xml|  5 ++---
 jdbc/pom.xml|  5 ++---
 kylin-it/pom.xml| 13 ++---
 pom.xml | 11 +--
 query/pom.xml   |  5 ++---
 server-base/pom.xml |  5 ++---
 server/pom.xml  |  5 ++---
 source-hive/pom.xml |  5 ++---
 source-kafka/pom.xml|  5 ++---
 storage-hbase/pom.xml   |  8 +++-
 tomcat-ext/pom.xml  |  6 ++
 tool/pom.xml|  5 ++---
 21 files changed, 50 insertions(+), 73 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bf6b1eb0/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index e6f83a8..17716bc 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -17,12 +17,11 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
 kylin
 org.apache.kylin
-1.6.0-SNAPSHOT
+1.6.0
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/bf6b1eb0/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 177b933..5185962 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 atopcalcite
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.0
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/bf6b1eb0/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 780fd82..a2ca10f 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 kylin-core-common
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.6.0
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/bf6b1eb0/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 39bba59..7cfcf9f 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0

kylin git commit: fix unit test

2016-11-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2195 66a7167d4 -> ce274c68d


fix unit test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce274c68
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce274c68
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce274c68

Branch: refs/heads/KYLIN-2195
Commit: ce274c68d04e7a4a139525d5e5fdab8fb16a276a
Parents: 66a7167
Author: Li Yang 
Authored: Wed Nov 23 11:55:48 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 23 11:55:48 2016 +0800

--
 .../src/test/java/org/apache/kylin/common/KylinConfigTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ce274c68/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java 
b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index 886d8ee..9257718 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -56,7 +56,7 @@ public class KylinConfigTest extends 
LocalFileMetadataTestCase {
 @Test
 public void testBackwardCompatibility() {
 KylinConfig config = KylinConfig.getInstanceFromEnv();
-final String oldk = "kylin.test.bcc.new.key";
+final String oldk = "kylin.test.bcc.old.key";
 final String newk = "kylin.test.bcc.new.key";
 
 assertNull(config.getOptional(oldk));

[5/8] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java
index db331d5..fb0db8c 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java
@@ -1,79 +1,79 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.ISourceAware;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- *
- */
-public class StreamingCubeRule implements IValidatorRule {
-
-/*
- * (non-Javadoc)
- * 
- * @see
- * 
org.apache.kylin.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , org.apache.kylin.metadata.validation.ValidateContext)
- */
-@Override
-public void validate(CubeDesc cube, ValidateContext context) {
-DataModelDesc model = cube.getModel();
-
-if (model.getRootFactTable().getTableDesc().getSourceType() != 
ISourceAware.ID_STREAMING) {
-return;
-}
-
-if (model.getLookupTables().size() > 0) {
-context.addResult(ResultLevel.ERROR, "Streaming Cube doesn't 
support star-schema so far; only one fact table is allowed.");
-return;
-}
-
-if (model.getPartitionDesc() == null || 
model.getPartitionDesc().getPartitionDateColumn() == null) {
-context.addResult(ResultLevel.ERROR, "Must define a partition 
column.");
-return;
-}
-
-final TblColRef partitionCol = 
model.getPartitionDesc().getPartitionDateColumnRef();
-boolean found = false;
-for (DimensionDesc dimensionDesc : cube.getDimensions()) {
-for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
-if (dimCol.equals(partitionCol)) {
-found = true;
-break;
-}
-}
-}
-
-if (found == false) {
-context.addResult(ResultLevel.ERROR, "Partition column '" + 
partitionCol + "' isn't in dimension list.");
-return;
-}
-
-}
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.model.validation.rule;
+
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ResultLevel;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.ISourceAware;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ *
+ */
+public class

[7/8] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index db02f34..db9fc80 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -1,246 +1,246 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-
-/**
- * @author yangli9
- */
-public class ByteArray implements Comparable, Serializable {
-
-private static final long serialVersionUID = 1L;
-
-public static final ByteArray EMPTY = new ImmutableByteArray();
-
-public static ByteArray allocate(int length) {
-return new ByteArray(new byte[length]);
-}
-
-public static ByteArray copyOf(byte[] array, int offset, int length) {
-byte[] space = new byte[length];
-System.arraycopy(array, offset, space, 0, length);
-return new ByteArray(space, 0, length);
-}
-
-// 

-
-private byte[] data;
-private int offset;
-private int length;
-
-public ByteArray() {
-this(null, 0, 0);
-}
-
-public ByteArray(int capacity) {
-this(new byte[capacity], 0, capacity);
-}
-
-public ByteArray(byte[] data) {
-this(data, 0, data == null ? 0 : data.length);
-}
-
-public ByteArray(byte[] data, int offset, int length) {
-this.data = data;
-this.offset = offset;
-this.length = length;
-}
-
-public byte[] array() {
-return data;
-}
-
-public int offset() {
-return offset;
-}
-
-public int length() {
-return length;
-}
-
-public void set(byte[] array) {
-set(array, 0, array.length);
-}
-
-public void set(byte[] array, int offset, int length) {
-this.data = array;
-this.offset = offset;
-this.length = length;
-}
-
-public void set(ByteArray o) {
-set(o.data, o.offset, o.length);
-}
-
-public void set(int offset, int length) {
-this.offset = offset;
-this.length = length;
-}
-
-public void setLength(int length) {
-this.length = length;
-}
-
-public ByteArray copy() {
-ByteArray copy;
-if (data != null) {
-copy = new ByteArray(length);
-} else {
-copy = new ByteArray(null);
-}
-copy.copyFrom(this);
-return copy;
-}
-
-//notice this will have a length header
-public void exportData(ByteBuffer out) {
-BytesUtil.writeByteArray(this.data, this.offset, this.length, out);
-}
-
-public static ByteArray importData(ByteBuffer in) {
-byte[] bytes = BytesUtil.readByteArray(in);
-return new ByteArray(bytes);
-}
-
-public void copyFrom(ByteArray other) {
-if (other.data != null) {
-System.arraycopy(other.array(), other.offset, data, offset, 
other.length);
-}
-this.length = other.length;
-}
-
-public ByteBuffer asBuffer() {
-if (data == null)
-return null;
-else if (offset == 0 && length == data.length)
-return ByteBuffer.wrap(data);
-else
-return ByteBuffer.wrap(data, offset, length).slice();
-}
-
-public byte[] toBytes() {
-return Bytes.copy(this.array(), this.offset(), this.length());
-}
-
-@Override
-public int hashCode() {
-if (data == null) {
-return 0;
-} else {
-if (length <= Bytes.SIZEOF_LONG && length > 0) {
-// to avoid hash collision of byte arrays those are converted 
from nearby integers/longs, which is the case for kylin dictionary
-long value = BytesUtil.readLong(data, offset, length);
-return (int) (value ^ (value

[8/8] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-22 Thread liyang

KYLIN-2195 All code changes, ready for test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/66a7167d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/66a7167d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/66a7167d

Branch: refs/heads/KYLIN-2195
Commit: 66a7167d4e4dbb1a67106e961b746b23283b04db
Parents: 53b5a27
Author: Li Yang 
Authored: Wed Nov 23 11:49:20 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 23 11:49:20 2016 +0800

--
 build/bin/check-env.sh  |2 +-
 build/bin/find-hive-dependency.sh   |4 +-
 build/bin/sample.sh |8 +-
 build/conf/kylin.properties |   94 +-
 .../common/BackwardCompatibilityConfig.java |   21 +-
 .../apache/kylin/common/KylinConfigBase.java|  272 ++---
 .../common/persistence/JsonSerializer.java  |   94 +-
 .../kylin/common/persistence/Serializer.java|   70 +-
 .../common/persistence/StorageException.java|   76 +-
 .../kylin/common/restclient/RestClient.java |  222 ++--
 .../org/apache/kylin/common/util/Array.java |  164 +--
 .../org/apache/kylin/common/util/ByteArray.java |  492 -
 .../kylin/common/util/BytesSerializer.java  |   70 +-
 .../apache/kylin/common/util/BytesSplitter.java |  250 ++---
 .../kylin/common/util/CliCommandExecutor.java   |  298 ++---
 .../apache/kylin/common/util/MailService.java   |2 +-
 .../apache/kylin/common/util/ShardingHash.java  |  134 +--
 .../kylin-backward-compatibility.properties |  358 +++---
 .../apache/kylin/common/KylinConfigTest.java|2 +-
 .../kylin/common/util/MailServiceTest.java  |2 +-
 .../cube/common/FuzzyValueCombination.java  |  260 ++---
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java |  286 ++---
 .../org/apache/kylin/cube/kv/RowConstants.java  |   94 +-
 .../apache/kylin/cube/kv/RowKeyColumnIO.java|  126 +--
 .../apache/kylin/cube/kv/RowKeyColumnOrder.java |  216 ++--
 .../org/apache/kylin/cube/model/CubeDesc.java   |2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |  322 +++---
 .../kylin/cube/model/v1_4_0/RowKeyColDesc.java  |  184 ++--
 .../cube/model/validation/IValidatorRule.java   |   54 +-
 .../cube/model/validation/ResultLevel.java  |   76 +-
 .../cube/model/validation/ValidateContext.java  |  200 ++--
 .../validation/rule/AggregationGroupRule.java   |  400 +++
 .../model/validation/rule/FunctionRule.java |  404 +++
 .../model/validation/rule/RowKeyAttrRule.java   |  110 +-
 .../validation/rule/StreamingCubeRule.java  |  158 +--
 .../kylin/cube/AggregationGroupRuleTest.java|  244 ++---
 .../org/apache/kylin/cube/CubeDescTest.java |6 +-
 .../apache/kylin/cube/CubeManagerCacheTest.java |  156 +--
 .../apache/kylin/cube/RowKeyAttrRuleTest.java   |  120 +--
 .../metadata/measure/MeasureCodecTest.java  |  182 ++--
 .../kylin/dict/DictionaryInfoSerializer.java|  160 +--
 .../kylin/dict/TrieDictionaryBuilder.java   | 1020 +-
 .../kylin/dict/lookup/LookupStringTable.java|  220 ++--
 .../apache/kylin/dict/lookup/LookupTable.java   |  356 +++---
 .../kylin/dict/lookup/SnapshotManager.java  |  370 +++
 .../apache/kylin/dict/lookup/SnapshotTable.java |  570 +-
 .../dict/lookup/SnapshotTableSerializer.java|  158 +--
 .../kylin/dict/AppendTrieDictionaryTest.java|2 +-
 .../impl/threadpool/DistributedScheduler.java   |6 +-
 .../kylin/metadata/MetadataConstants.java   |  104 +-
 .../kylin/metadata/cachesync/Broadcaster.java   |2 +-
 .../kylin/metadata/model/JoinTableDesc.java |  124 +--
 .../org/apache/kylin/metadata/tuple/ITuple.java |   86 +-
 .../apache/kylin/storage/StorageContext.java|2 +-
 .../translate/DerivedFilterTranslator.java  |  400 +++
 .../translate/FuzzyValueCombination.java|  264 ++---
 .../apache/kylin/engine/mr/DFSFileTable.java|  198 ++--
 .../kylin/engine/mr/DFSFileTableReader.java |  504 -
 .../engine/mr/steps/FactDistinctColumnsJob.java |  318 +++---
 .../apache/kylin/engine/mr/TableReaderTest.java |   92 +-
 .../test_case_data/localmeta/kylin.properties   |   52 +-
 .../test_case_data/localmeta/kylin_job_conf.xml |  134 +--
 examples/test_case_data/sandbox/hive-site.xml   |6 +-
 .../test_case_data/sandbox/kylin.properties |   64 +-
 .../test_case_data/sandbox/kylin_job_conf.xml   |  156 +--
 .../sandbox/kylin_job_conf_inmem.xml|  182 ++--
 .../resources/org-apache-kylin-jdbc.properties  |   54 +-
 .../kylin/job/BaseTestDistributedScheduler.java |4 +-
 .../kylin/provision/BuildCubeWithEngine.java|2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|4 +-
 kylin-it/src/test/resources/logging.properties  |   44 +-
 .../aggregate/DimCountDistinctAggFuncTest.java  |4

[4/8] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 916c369..598865b 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -1,512 +1,512 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.IdentityHashMap;
-import java.util.LinkedList;
-
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- * Builds a dictionary using Trie structure. All values are taken in byte[] 
form
- * and organized in a Trie with ordering. Then numeric IDs are assigned in
- * sequence.
- * 
- * @author yangli9
- */
-public class TrieDictionaryBuilder {
-
-private static final int _2GB = 20;
-
-public static class Node {
-public byte[] part;
-public boolean isEndOfValue;
-public ArrayList children;
-
-public int nValuesBeneath; // only present after stats()
-
-Node(byte[] value, boolean isEndOfValue) {
-reset(value, isEndOfValue);
-}
-
-Node(byte[] value, boolean isEndOfValue, ArrayList children) {
-reset(value, isEndOfValue, children);
-}
-
-void reset(byte[] value, boolean isEndOfValue) {
-reset(value, isEndOfValue, new ArrayList());
-}
-
-void reset(byte[] value, boolean isEndOfValue, ArrayList 
children) {
-this.part = value;
-this.isEndOfValue = isEndOfValue;
-this.children = children;
-}
-}
-
-public static interface Visitor {
-void visit(Node n, int level);
-}
-
-// 

-
-private Node root;
-private BytesConverter bytesConverter;
-
-public TrieDictionaryBuilder(BytesConverter bytesConverter) {
-this.root = new Node(new byte[0], false);
-this.bytesConverter = bytesConverter;
-}
-
-public void addValue(T value) {
-addValue(bytesConverter.convertToBytes(value));
-}
-
-// add a converted value (given in byte[] format), use with care, for 
internal only
-void addValue(byte[] value) {
-addValueR(root, value, 0);
-}
-
-private void addValueR(Node node, byte[] value, int start) {
-// match the value part of current node
-int i = 0, j = start;
-int n = node.part.length, nn = value.length;
-int comp = 0;
-for (; i < n && j < nn; i++, j++) {
-comp = BytesUtil.compareByteUnsigned(node.part[i], value[j]);
-if (comp != 0)
-break;
-}
-
-if (j == nn) {
-// if value fully matched within the current node
-if (i == n) {
-// if equals to current node, just mark end of value
-node.isEndOfValue = true;
-} else {
-// otherwise, split the current node into two
-Node c = new Node(BytesUtil.subarray(node.part, i, n), 
node.isEndOfValue, node.children);
-node.reset(BytesUtil.subarray(node.part, 0, i), true);
-node.children.add(c);
-}
-return;
-}
-
-// if partially matched the current, split the current node, add the 
new value, make a 3-way
-if (i < n) {
-Node c1 = new Node(BytesUtil.subarray(node.part, i, n), 
node.isEndOfValue, node.children);
-Node c2 = new Node(BytesUtil.subarray(value, j, nn), true);
-node.reset(BytesUtil.subarray(node.part, 0, i), false);
-if (comp < 0) {
-

[3/8] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index db1a170..f2edcb4 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -1,285 +1,285 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.File;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import com.google.common.base.Strings;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.StringBytesConverter;
-import org.apache.kylin.dict.TrieDictionary;
-import org.apache.kylin.dict.TrieDictionaryBuilder;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.source.ReadableTable;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author yangli9
- */
-@SuppressWarnings("serial")
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
-public class SnapshotTable extends RootPersistentEntity implements 
ReadableTable {
-
-@JsonProperty("tableName")
-private String tableName;
-@JsonProperty("signature")
-private TableSignature signature;
-@JsonProperty("useDictionary")
-private boolean useDictionary;
-
-private ArrayList rowIndices;
-private Dictionary dict;
-
-// default constructor for JSON serialization
-public SnapshotTable() {
-}
-
-SnapshotTable(ReadableTable table, String tableName) throws IOException {
-this.tableName = tableName;
-this.signature = table.getSignature();
-this.useDictionary = true;
-}
-
-public void takeSnapshot(ReadableTable table, TableDesc tableDesc) throws 
IOException {
-this.signature = table.getSignature();
-
-int maxIndex = tableDesc.getMaxColumnIndex();
-
-TrieDictionaryBuilder b = new 
TrieDictionaryBuilder(new StringBytesConverter());
-
-TableReader reader = table.getReader();
-try {
-while (reader.next()) {
-String[] row = reader.getRow();
-if (row.length <= maxIndex) {
-throw new IllegalStateException("Bad hive table row, " + 
tableDesc + " expect " + (maxIndex + 1) + " columns, but got " + 
Arrays.toString(row));
-}
-for (ColumnDesc column : tableDesc.getColumns()) {
-String cell = row[column.getZeroBasedIndex()];
-if (cell != null)
-b.addValue(cell);
-}
-}
-} finally {
-IOUtils.closeQuietly(reader);
-}
-
-this.dict = b.build(0);
-
-ArrayList allRowIndices = new ArrayList();
-reader = table.getReader();
-try {
-while (reader.next()) {
-String[] row = reader.getRow();
-int[] rowIndex = new int[tableDesc.getColumnCount()];
-for (ColumnDesc column : tableDesc.getColumns()) {
-rowIndex[column.getZeroBasedIndex()] = 
dict.getIdFromValue(row[column.getZeroBasedIndex()]);
-}
-allRowIndices.add(rowIndex);
-}
-} finally {
-

[2/8] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/66a7167d/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
index 173c908..67fedbd 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
@@ -1,252 +1,252 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr;
-
-import java.io.BufferedReader;
-import java.io.Closeable;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.StringSplitter;
-import org.apache.kylin.source.ReadableTable.TableReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Tables are typically CSV or SEQ file.
- * 
- * @author yangli9
- */
-public class DFSFileTableReader implements TableReader {
-
-private static final Logger logger = 
LoggerFactory.getLogger(DFSFileTableReader.class);
-private static final char CSV_QUOTE = '"';
-private static final String[] DETECT_DELIMS = new String[] { "\177", "|", 
"\t", "," };
-
-private String filePath;
-private String delim;
-private List readerList;
-
-private String curLine;
-private String[] curColumns;
-private int expectedColumnNumber = -1; // helps delimiter detection
-
-public DFSFileTableReader(String filePath, int expectedColumnNumber) 
throws IOException {
-this(filePath, DFSFileTable.DELIM_AUTO, expectedColumnNumber);
-}
-
-public DFSFileTableReader(String filePath, String delim, int 
expectedColumnNumber) throws IOException {
-filePath = HadoopUtil.fixWindowsPath(filePath);
-this.filePath = filePath;
-this.delim = delim;
-this.expectedColumnNumber = expectedColumnNumber;
-this.readerList = new ArrayList();
-
-FileSystem fs = HadoopUtil.getFileSystem(filePath);
-
-ArrayList allFiles = new ArrayList<>();
-FileStatus status = fs.getFileStatus(new Path(filePath));
-if (status.isFile()) {
-allFiles.add(status);
-} else {
-FileStatus[] listStatus = fs.listStatus(new Path(filePath));
-allFiles.addAll(Arrays.asList(listStatus));
-}
-
-try {
-for (FileStatus f : allFiles) {
-RowReader rowReader = new 
SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, f.getPath().toString());
-this.readerList.add(rowReader);
-}
-} catch (IOException e) {
-if (isExceptionSayingNotSeqFile(e) == false)
-throw e;
-
-this.readerList = new ArrayList();
-for (FileStatus f : allFiles) {
-RowReader rowReader = new CsvRowReader(fs, 
f.getPath().toString());
-this.readerList.add(rowReader);
-}
-}
-}
-
-private boolean isExceptionSayingNotSeqFile(IOException e) {
-if (e.getMessage() != null && e.getMessage().contains("not a 
SequenceFile"))
-return true;
-
-if (e instanceof EOFException) // in case the file is very very small
-return true;
-
-return false;
-}
-
-@Override
-public boolean next() throws IOException {
-int curReaderIndex = -1;
-

[kylin] Git Push Summary

2016-11-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2195 [created] 53b5a2752

[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-22 Thread lidong

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2900745
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2900745
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2900745

Branch: refs/heads/yang21-cdh5.7
Commit: e29007457f5668ad8695c4c57631b3aabfc98327
Parents: 6020f87
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Wed Nov 23 11:07:07 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2900745/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2900745/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2900745/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2900745/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-22 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6020f87d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6020f87d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6020f87d

Branch: refs/heads/yang21-cdh5.7
Commit: 6020f87db86ef06e23b314cc6e853b415481a987
Parents: 150bab6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Nov 23 11:05:31 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6020f87d/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-22 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6020f87d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6020f87d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6020f87d

Branch: refs/heads/yang21-hbase1.x
Commit: 6020f87db86ef06e23b314cc6e853b415481a987
Parents: 150bab6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Nov 23 11:05:31 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6020f87d/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2016-11-22 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x f390b341d -> 6020f87db (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/6020f87d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6020f87d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-

[kylin] Git Push Summary

2016-11-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2197 [deleted] 44334b467

kylin git commit: KYLIN-2195 Move backward-compatibility properties to common resources

2016-11-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 747337ae4 -> 53b5a2752


KYLIN-2195 Move backward-compatibility properties to common resources


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/53b5a275
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/53b5a275
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/53b5a275

Branch: refs/heads/master
Commit: 53b5a2752bfb282445572e2dbabe6cee93cf404c
Parents: 747337a
Author: Yang Li 
Authored: Wed Nov 23 08:23:02 2016 +0800
Committer: Yang Li 
Committed: Wed Nov 23 08:23:22 2016 +0800

--
 .../common/BackwardCompatibilityConfig.java | 128 ++---
 .../kylin-backward-compatibility.properties | 186 +++
 .../apache/kylin/common/KylinConfigTest.java|  15 +-
 .../kylin-backward-compatibility.properties | 186 ---
 4 files changed, 291 insertions(+), 224 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/53b5a275/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
 
b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
index 93c834e..0239af0 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
@@ -18,13 +18,15 @@
 
 package org.apache.kylin.common;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintWriter;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
-import java.util.Set;
+import java.util.Stack;
 
 import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
@@ -35,23 +37,29 @@ import com.google.common.collect.Maps;
 public class BackwardCompatibilityConfig {
 
 private static final Logger logger = 
LoggerFactory.getLogger(BackwardCompatibilityConfig.class);
-
+
+private static final String KYLIN_BACKWARD_COMPATIBILITY = 
"kylin-backward-compatibility";
+
 private final Map old2new = Maps.newConcurrentMap();
-
+private final Map old2newPrefix = Maps.newConcurrentMap();
+
 public BackwardCompatibilityConfig() {
-InputStream is = 
Thread.currentThread().getContextClassLoader().getResourceAsStream("kylin-backward-compatibility.properties");
-init(is);
+ClassLoader loader = Thread.currentThread().getContextClassLoader();
+init(loader.getResourceAsStream(KYLIN_BACKWARD_COMPATIBILITY + 
".properties"));
+for (int i = 0; i < 10; i++) {
+init(loader.getResourceAsStream(KYLIN_BACKWARD_COMPATIBILITY + (i) 
+ ".properties"));
+}
 }
-
+
 // for test
 BackwardCompatibilityConfig(InputStream is) {
 init(is);
 }
-
+
 private void init(InputStream is) {
 if (is == null)
 return;
-
+
 Properties props = new Properties();
 try {
 props.load(is);
@@ -60,50 +68,116 @@ public class BackwardCompatibilityConfig {
 } finally {
 IOUtils.closeQuietly(is);
 }
-
+
 for (Entry kv : props.entrySet()) {
-old2new.put((String) kv.getKey(), (String) kv.getValue());
+String key = (String) kv.getKey();
+String value = (String) kv.getValue();
+if (value.endsWith("."))
+old2newPrefix.put(key, value);
+else
+old2new.put(key, value);
 }
 }
-
+
 public String check(String key) {
 String newKey = old2new.get(key);
 if (newKey != null) {
 logger.warn("Config '{}' is deprecated, use '{}' instead", key, 
newKey);
 return newKey;
-} else {
-return key;
 }
+
+for (String oldPrefix : old2newPrefix.keySet()) {
+if (key.startsWith(oldPrefix)) {
+String newPrefix = old2newPrefix.get(oldPrefix);
+newKey = newPrefix + key.substring(oldPrefix.length() + 1);
+logger.warn("Config '{}' is deprecated, use '{}' instead", 
key, newKey);
+return newKey;
+}
+}
+
+return key;
 }
-
+
 public Map check(Map props) {
-if (containsOldKey(props.keySet()) == false)
-return props;
-
 LinkedHashMap result = new LinkedHashMap<>();
 for (Entry kv :

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2016-11-22 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x e1b83470b -> f390b341d (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/f390b341/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f390b341/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-22 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f390b341
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f390b341
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f390b341

Branch: refs/heads/yang21-cdh5.7
Commit: f390b341d25f49ef3efe34f74f5532699db8292e
Parents: 150bab6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Tue Nov 22 23:14:59 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f390b341/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2016-11-22 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 ac2d7c41d -> d54e8ba43 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/f390b341/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f390b341/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-22 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f390b341
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f390b341
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f390b341

Branch: refs/heads/yang21-hbase1.x
Commit: f390b341d25f49ef3efe34f74f5532699db8292e
Parents: 150bab6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Tue Nov 22 23:14:59 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f390b341/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[2/2] kylin git commit: KYLIN-2133: Check web server port availability when startup

2016-11-22 Thread shaofengshi

KYLIN-2133: Check web server port availability when startup

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/747337ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/747337ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/747337ae

Branch: refs/heads/master
Commit: 747337ae405b7f21aa711a66e2f58bf6547f8f67
Parents: 657aa31
Author: Yiming Liu 
Authored: Thu Oct 27 15:24:15 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 22 22:42:16 2016 +0800

--
 build/bin/kylin.sh | 9 +
 1 file changed, 9 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/747337ae/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index dedb552..fff6227 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -94,6 +94,15 @@ then
 fi
 verbose "kylin.rest.address is set to ${kylin_rest_address}"
 
+kylin_rest_address_arr=(${kylin_rest_address//;/ })
+nc -z -w 5 ${kylin_rest_address_arr[0]} ${kylin_rest_address_arr[1]} 
1>/dev/null 2>&1; nc_result=$?
+if [ $nc_result -eq 0 ]; then
+echo "port ${kylin_rest_address} is not available, could not start 
Kylin"
+exit 1
+else
+echo "port ${kylin_rest_address} is available"
+fi
+
 #debug if encounter NoClassDefError
 verbose "kylin classpath is: $(hbase classpath)"

[1/2] kylin git commit: KYLIN-2196: enable tomcat parallel class loader

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master c8dfa13e6 -> 747337ae4


KYLIN-2196: enable tomcat parallel class loader

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/657aa31b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/657aa31b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/657aa31b

Branch: refs/heads/master
Commit: 657aa31b29b1b22d96c7ef121276ffa60ba9ea5d
Parents: c8dfa13
Author: Yiming Liu 
Authored: Wed Nov 16 13:06:56 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 22 22:41:11 2016 +0800

--
 .../java/org/apache/kylin/ext/CustomizedWebappClassloader.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/657aa31b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
--
diff --git 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
index d97d583..5116849 100644
--- 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
+++ 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
@@ -18,14 +18,14 @@
 
 package org.apache.kylin.ext;
 
-import org.apache.catalina.loader.WebappClassLoader;
+import org.apache.catalina.loader.ParallelWebappClassLoader;
 
 /**
- * simple extension to standard WebappClassLoader
+ * simple extension to standard ParallelWebappClassLoader
  * the only difference is that CustomizedWebappClassloader is able to delegate 
more packages
  * to parent classloaders
  */
-public class CustomizedWebappClassloader extends WebappClassLoader {
+public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
 /**
  * Set of package names which are not allowed to be loaded from a webapp
  * class loader without delegating first.

kylin git commit: float data format func bug

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 2b7fe6106 -> c8dfa13e6


float data format func bug

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8dfa13e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8dfa13e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8dfa13e

Branch: refs/heads/master
Commit: c8dfa13e633517bbfde7125ce286429f643e5af6
Parents: 2b7fe61
Author: luguosheng <550175...@qq.com>
Authored: Tue Nov 22 18:50:52 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 22 21:43:54 2016 +0800

--
 webapp/app/js/utils/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c8dfa13e/webapp/app/js/utils/utils.js
--
diff --git a/webapp/app/js/utils/utils.js b/webapp/app/js/utils/utils.js
index aabcad5..9da6196 100644
--- a/webapp/app/js/utils/utils.js
+++ b/webapp/app/js/utils/utils.js
@@ -67,7 +67,7 @@ KylinApp.factory('VdmUtil', function ($modal, $timeout, 
$location, $anchorScroll
 
 SCToFloat:function(data){
   var resultValue = "";
-  if (data.indexOf('E') != -1){
+  if (data&('E') != -1){
 var regExp = new RegExp('^((\\d+.?\\d+)[Ee]{1}(\\d+))$', 'ig');
 var result = regExp.exec(data);
 var power = "";

kylin git commit: float data format func bug

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 33bb7c293 -> d675b7a06


float data format func bug

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d675b7a0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d675b7a0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d675b7a0

Branch: refs/heads/v1.6.0-rc2
Commit: d675b7a061ab924381999c0724e82f6e5b16fe8c
Parents: 33bb7c2
Author: luguosheng <550175...@qq.com>
Authored: Tue Nov 22 18:50:52 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 22 21:40:38 2016 +0800

--
 webapp/app/js/utils/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d675b7a0/webapp/app/js/utils/utils.js
--
diff --git a/webapp/app/js/utils/utils.js b/webapp/app/js/utils/utils.js
index d25c4e1..2e95342 100644
--- a/webapp/app/js/utils/utils.js
+++ b/webapp/app/js/utils/utils.js
@@ -67,7 +67,7 @@ KylinApp.factory('VdmUtil', function ($modal, $timeout, 
$location, $anchorScroll
 
 SCToFloat:function(data){
   var resultValue = "";
-  if (data.indexOf('E') != -1){
+  if (data&('E') != -1){
 var regExp = new RegExp('^((\\d+.?\\d+)[Ee]{1}(\\d+))$', 'ig');
 var result = regExp.exec(data);
 var power = "";

[04/10] kylin git commit: KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some useless MR files.

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b7fe610/engine-mr/src/test/resources/multi_file_str/data_1
--
diff --git a/engine-mr/src/test/resources/multi_file_str/data_1 
b/engine-mr/src/test/resources/multi_file_str/data_1
new file mode 100644
index 000..ee43541
--- /dev/null
+++ b/engine-mr/src/test/resources/multi_file_str/data_1
@@ -0,0 +1,1990 @@
+0005cc94-b007-4f31-90f0-4d80410d5e79
+005057a2-ef79-4b31-8480-8397e0f6e2bb
+007fbfec-dc89-4256-afa9-c52269e7a31e
+00b18cf2-53d0-4be1-80fd-bec6a47b46bc
+00d907e8-a459-4663-8b71-0da459be50e1
+00ef7481-3a29-40cc-b6f5-202a24b44a0c
+018c86b0-279b-40f4-9758-32d0a1a522c3
+01a07ce6-7fd4-4ae2-a430-12a03788199c
+01ae765e-226e-4642-b382-0c6cc2eca798
+01b1e026-a47b-4ff8-85f2-8de89334139e
+01fb060a-ae9b-47b2-831d-4c36b6c93b1e
+021426fa-84d6-4ddc-8974-15d908f5e601
+021dade6-1085-49bd-97bb-e4b223c8425c
+022b895c-a8ce-4a18-ba9d-be103b296771
+022d5709-23fa-464c-ad96-dd2fddf58e68
+025e3a2e-44af-4694-875d-6ca9de45e9a4
+02606751-63fc-417b-93fe-3fd3ec4942f4
+02a259a7-2b35-4dab-9857-964cae1b3630
+02a4fe44-3464-4de5-bdb0-e256fa71a2b7
+02bb8408-39a0-4818-add3-02f915782fb0
+02e36b48-46d8-49ac-9168-8490518f1922
+02e620e8-39ef-4ceb-ad69-d2cfe6d9b4b7
+0305fbd2-7c7b-42d8-9153-66e191a69efe
+0310efc2-5dbe-4586-beeb-d2480d4bbb1f
+031774dd-0b13-4279-b31a-462e6e9442bc
+03248fd7-83b4-4c24-b281-9b6c3811
+036195d3-1ce6-4e59-a8d9-36d0d35f248c
+03a25119-497a-48d5-9f50-b5a87aab84b9
+03af2273-fb4b-42d5-a9b8-b459a2d90c78
+03cdbdba-6923-4335-8cc2-b3b2317a6160
+03f0a4a3-07be-4f7b-867d-82885baefaeb
+03f6e287-f5ae-479c-8a55-56f74afed997
+041ae6d0-9bc7-4342-a440-7ab2a402a0dc
+041ae917-3697-44c1-a78d-a0799bb697e6
+04250b0a-acf7-4117-bab0-1d42b796d90d
+043fe4f6-5cd9-4cf1-9f95-b6b56f310ef9
+0440d813-998e-4e60-9f8f-50becf8066bf
+0443dc38-0193-4a23-9b66-b4cb6cbb555d
+04793483-e5ec-495d-a211-4026d7701b46
+04b56c59-c15d-41de-bea5-b1715b69b3c8
+04bf76e7-23b5-402d-9e0c-740ce8a3387d
+04c0f225-0897-479b-84eb-e3600baaf361
+04fdb9bb-0088-42a3-b4bf-cd91ccea2fe0
+050ddd3b-f8bd-4f5a-a839-182a97367aee
+055a3f59-14f8-4893-88f4-ecad22adc150
+057191c2-3f8b-48ed-b932-9ec2df13849e
+05c3a70d-6185-44d4-be48-307022efb1d5
+0640b40a-e418-4e90-8675-13458f256c20
+0658d229-9b4b-4559-9737-dd90b22f6a63
+06641f20-c6db-449b-8e4e-a4f2d62ca2fe
+0668c755-716a-4921-b05f-f20f60e6818a
+068e231b-d4e6-4785-8d31-f8f276b330bb
+06a4efa2-8217-45c9-9402-9a63f0efbec2
+06e942ab-c053-41a5-8686-237cdde18365
+06f00088-e14a-496e-9510-3d26cf3a5782
+071d0310-1e75-4b91-9d06-ad1ed1c39d46
+07254c89-5eda-424c-bc9b-f44e568f444c
+0731d06d-8c2f-4560-b5ee-9be7b7452ecf
+076b0aa7-c95d-40be-9a15-835b93b171bb
+076ffc77-c433-46ac-8089-cdb7bb08672a
+0789584e-3d3c-448f-9268-0c63f37dff70
+07a771f3-00b1-4c0b-baa8-23ce0dce16ab
+07c8fe67-dce8-4f19-92df-bbcef68de151
+07d4da44-06e0-4289-b422-33717482d0b5
+07efc40a-f27d-4ca6-a86a-fe720ff5246c
+0822873c-cae8-417a-9115-cd42443b74c8
+0849beac-aeb6-4e1e-a3fd-e22a394b2bc8
+08603138-d916-4c56-9979-23ddcdfc7d79
+088c4d06-edd5-4755-8ea7-0e1287b8bff2
+08d6bada-5adc-4a33-acc7-e07ec245343f
+08d82dd5-cf91-405e-8e6c-7cf12ace017e
+09067e02-617b-4ce3-8ac6-77fe43e9f782
+09200ea2-2c20-437b-91d9-ad22656148ac
+09503c81-fb15-45bf-b388-038fa9099879
+09679853-7f41-40e2-8d38-7fe326f7248f
+09de044d-1438-4fd1-a084-fd1d570988df
+0a3138fe-4be2-47d7-b17e-0dcbeea7f453
+0a350c27-eb95-4b4a-ab66-ae65ecefb15e
+0a4296da-bcd2-4b45-bd0c-93b7c9164c52
+0a578961-fe44-489a-a7d2-d68ba68bc8d9
+0a6bf3fa-594f-4658-b93e-316ea5022534
+0a6c416f-7122-4d72-b9d6-3d0669e4009b
+0a7a5b4b-1aa5-4732-a7b8-66ffc84eab4c
+0a8cf4ff-685b-44b8-a78e-16c0aca6db62
+0ab0fe90-fce9-4c9e-90e3-81f165b64e91
+0ac33843-5fca-4df9-86d4-e1dbc4ea92bd
+0af5a7ea-82ac-461c-a122-6f034a01cd95
+0b100b1d-487e-4a58-9f46-7a776b079dda
+0b3e7018-723c-4be0-8839-12d74a42fb46
+0b510dac-8009-4b3b-a8ba-4ef8b746f988
+0b61f652-7cfa-45a7-be3e-ad5eee5bf726
+0b7eb036-cc3d-4df7-b532-436c7366c369
+0b9820ab-e846-4b61-8b97-5def0503c9d2
+0baacd76-d04c-4f78-81a4-3c9e78e242bd
+0bb23b64-9a75-4f32-82dd-87f8c3e7cb6a
+0bc5f734-c638-4815-950d-a4b37d8eea25
+0bd64b0a-868d-4b92-a5a8-7d93aa5159ec
+0bdc03d0-73b5-41ff-80a6-f1c215cc46e5
+0c2d5a38-a98b-4a4a-b8ab-cf058a60fd7d
+0c338748-ac2e-488a-a365-b75685a0461d
+0c3765f0-2259-4987-b797-f88f35ea0873
+0c98b2dc-c5c4-46b5-b350-584d7bf6fa8b
+0cd4d27d-4d95-4d0b-aab8-7e28dcc830df
+0cdec55a-22f3-4aef-bacd-378c742eccf9
+0cf1c496-7d40-4c8e-ab0f-9aabd69e78dd
+0d1295ca-6d5e-4617-8017-c7a6243cc0bd
+0d873bb7-1a0b-42df-82d0-c15201e8d1a3
+0da55227-7a69-4ed0-bd71-67cebe5b4b47
+0dd9c9c5-71ee-4523-bfaa-64f3c82b45c0
+0de68325-19c4-4668-89cb-a21818613e5e
+0e3e8c17-60c6-4dda-8648-b86f881c93ac
+0e6178d5-87de-422e-9ab0-4668c25fd398
+0edb80b3-d3d4-40a8-8d0e-839939aca2e1
+0ee1de00-5e13-4af7-9e7f-4e32f140d774
+0f4efb00-0888-4e30-bd23-f1fa0f87f80b
+0f6f7f5d-64a6-4406-a21a-0071b8313b9e
+0fa10b55-cb5b-4974-b1f9-8ab3a1e7226c
+0fb30e28-cac7-49d1-a649-8d04d6385747
+0fd28d53-6858-4902-8ba7-b00351a8d92a
+0fe86888-df1d-4605-a9da-73882413e983

[05/10] kylin git commit: KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some useless MR files.

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b7fe610/engine-mr/src/test/resources/multi_file_str/data_0
--
diff --git a/engine-mr/src/test/resources/multi_file_str/data_0 
b/engine-mr/src/test/resources/multi_file_str/data_0
new file mode 100644
index 000..89dd2b3
--- /dev/null
+++ b/engine-mr/src/test/resources/multi_file_str/data_0
@@ -0,0 +1,2029 @@
+0018bb26-1649-404d-a8b9-c79a505ed162
+001eb1f5-96f4-4031-92d3-8ecee8286c31
+00464de2-6452-4b42-80c8-7fa9e2e25dab
+0080cefc-f819-4679-84be-9a1ec3c254ca
+008472d0-dbb0-4a9e-9bbe-a285698f5996
+009a99e8-415e-4a73-a352-4acab798cf80
+00defa41-59cd-4079-a0c4-3bf88fb4d2f7
+0113c6c8-396a-402e-86e6-2cdcb6623be8
+0121d713-0519-4576-9cb6-e58f26d036af
+0141bd03-3548-4578-8850-f96f6f4ec27a
+01461b14-85ef-4283-8ff8-fc670c6d8e85
+01501d16-ff3c-4fef-a17a-66eaef23878c
+01524b01-7839-4022-b04e-74b1928f8fef
+0153828e-2028-4dd2-9ff3-7465ecebaac0
+01582c51-a738-4129-be8a-b82d7d6999c2
+01687978-951a-4fea-93a9-990bdaf378ef
+0190c832-0f82-4993-8235-6343774d2367
+01a066cb-160c-43d8-88d4-b1f447239040
+01c462fa-5ef0-4ecf-9e2c-fa356f3b83dd
+01eedbe6-b246-49e4-8761-86b3b1dea82c
+01f0bb43-620f-43f7-b07e-a032eca9d133
+022bc52b-0f8e-4b02-ab04-79012965ec3e
+022dd8e1-19e0-4c3d-b9f8-23ce0485af8a
+02379ebb-6d16-427c-a4bc-4f7707913ea1
+029b270d-9454-42fe-a7a3-f0c127b89fe9
+02a3f086-dfc3-4c35-827b-ff11f48c068e
+02d6d848-f3e1-4ad5-a7d5-4874e963c016
+0308f18d-d796-4745-bf10-46a23be3087b
+037c7649-393b-4150-a922-fa6b3488509e
+039bc304-4ef4-4480-a347-6f863e6c5d23
+039efd7f-ec8f-4716-bfc0-b254d6e5e88f
+03b52c5a-8bb6-4561-ab4d-7d491f8f4838
+03d69ddc-1eb3-4038-8c0c-51eb731db079
+04017bfd-32ad-4040-81aa-aa47206bb94f
+042119f5-10f4-4e8d-9626-871b33f063f4
+0428bca6-368d-4d58-babb-306179b2248a
+045ea515-b9d4-4738-8422-7c2ea163b328
+0467c216-7639-4c1d-a0a9-39f9a10dff6f
+048a183c-a958-47bf-aa45-25336aa5674d
+04fc427a-0b86-447c-96e7-8ddd1cc7569e
+050e5141-d731-417f-9793-469ed3a38d39
+051ab944-932d-4837-b2a4-363ad0c26e3c
+0528565e-7fd0-4dff-b8a9-66ade9691a8d
+05329edc-f1ad-43de-b01a-49a4db1f2d27
+058259f2-d7c1-4c54-8ec4-978677d71aa7
+059a0b69-8c64-4598-8dd2-dc72e3ef1d21
+05af50a4-9ff1-4673-9b10-db82732ae512
+05ca0b95-b68a-4076-b14b-de38694ac4d7
+05d28038-94ec-4950-b251-7b28aff6d73c
+05f3d13d-be28-4351-ba0f-28d7bf87710b
+05f607b9-cc8f-45e7-a7e6-7a8d93a94960
+05fc5cfe-5c0d-46b3-b448-1a7c2be69a33
+06471838--45b8-af74-508742d12ecd
+0667a812-bc2e-4a00-845d-64a85bb81503
+0681052c-e3a8-45ae-8f96-f8efcd9891cf
+06c59613-09f9-4bff-8bca-0e9e2962ea3e
+0719f720-20f5-4a22-8c03-888b03734b1e
+071c0fd1-1be3-46bc-8117-a5d4292ffb02
+07857916-cbff-4122-9c4c-da5d7c906c59
+07bca0a0-7405-4c75-98bc-a04dbdce261c
+07fe55a6-0071-4d03-b141-d4333d5695eb
+081b029d-8023-4825-805e-c9bfdad93ce4
+083fa003-02e9-4e5e-83f1-d7768ffbb539
+0851fcc1-919f-472a-ae79-209bc64371e6
+085233ed-547e-478f-99a4-41b2472b983f
+0877cca8-e95b-4e7f-bdfd-c3e8742f8140
+089f14fd-64b4-42a8-9fe1-603b3e3c8093
+090b9228-57c1-4f2d-b3ab-8d4a624c6571
+091cf088-6947-4929-b0da-5f4b401bad13
+092832d1-d672-43e8-9a67-6d46423a94f7
+0929094e-42f7-487f-8b2e-2e175e76ee6f
+0946e299-7a4d-402c-9c55-e6638782a23b
+09e1f414-5172-4d02-a8a5-86b12ccc5857
+09f6ffb4-a27a-4707-97b8-9364c495fd2c
+0a0483d4-11e1-43e8-8633-9ac9987e712a
+0a4c6743-0a3f-4cf3-ac50-48ec806f8800
+0a667611-f3aa-4c42-a9a4-388f746abc78
+0a7e409d-63ba-43dd-9651-a8809e625102
+0ab2b943-e09f-4b7a-aa6b-e319b9a4b8a8
+0ae2cead-abb0-4b86-97e3-e43d1dd454b3
+0af4329c-94c1-4a3a-b560-2f964883a88d
+0b226411-bb53-4c7a-92fa-1e2d3c7010d9
+0b296ac4-a698-4db8-9d46-5222c157e9fc
+0b685786-3b26-46ce-9444-a2ed74eab290
+0c19d519-086d-4c6f-9002-b81cad3996dd
+0c1b9859-17dd-4eb6-9dcb-162fba753b94
+0c3d7db2-5636-40d1-b5ed-d61ee557c99b
+0c471a75-6aec-4d7e-b728-b3f12596ebd2
+0c601fb1-5fdb-4986-a418-c515b64d6512
+0c6cbddd-c614-4e85-85b7-e9076dc2275f
+0c8d46f1-4c53-4ad6-ae5f-6183aa4123ec
+0c90f498-281b-4f04-b797-bfacd3f65ea1
+0c98e617-1b53-4402-a3ba-1392ac919113
+0cb95a98-f6b6-4ac9-bacf-3a3238b51e92
+0cda3414-38b1-49c8-9421-071ff259a388
+0ce90e93-4cb6-4d78-b919-fca7b2789534
+0cff1d3d-357c-4b69-877f-61051511915e
+0d0d5580-a0c1-4769-bd02-d7344216fbb6
+0d15b496-fb0b-4bdc-a808-03e567a8e8b3
+0d22693e-c76e-486b-9e95-9349a3553397
+0d6940fd-04d3-4464-9f13-6e89b0a0319b
+0d7d420c-8fa5-4efc-a146-1e9413ccd460
+0db484e9-d117-4450-8b5b-879b94587ac2
+0dc3e086-529d-4289-ad3c-4fa469940290
+0ded4753-81f4-458e-a661-a664165b0fd4
+0e0624ed-f8fa-4db8-b61d-d43cb0acd1ed
+0e26c5d0-b5ae-4304-9c9a-2b25df16c3d5
+0e49a979-46bc-4674-93b7-78ed98e582c1
+0e7af5c9-0c89-42af-b8a7-924897590277
+0e8f53bd-19c7-4fb2-a361-37f3fc47a5a2
+0eaeeb80-7c82-41f1-8a4c-62c77921ad50
+0f28de8f-a5c1-494e-a78e-06ae076ed87d
+0f384c24-19b0-4cc0-add2-670891da674f
+0f4a19ac-7d34-45da-b59a-11418bc96495
+0f4e9e96-601e-4e98-a0cc-71e20f95fc44
+0fdc6899-06da-4dda-ad00-bb12aa8fcc3c
+0ff80635-4ae8-47c0-8802-d3203239c5b2
+10035ab5-e72c-45fc-9a0c-609f89dbda9c
+10194e1f-2d29-4106-8b61-187721c543a9
+10316ab9-17bd-4005-8de2-de7315353a8f

[01/10] kylin git commit: KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some useless MR files.

2016-11-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 79cb59cfd -> 2b7fe6106


http://git-wip-us.apache.org/repos/asf/kylin/blob/2b7fe610/engine-mr/src/test/resources/multi_file_str/data_4
--
diff --git a/engine-mr/src/test/resources/multi_file_str/data_4 
b/engine-mr/src/test/resources/multi_file_str/data_4
new file mode 100644
index 000..7c0afd3
--- /dev/null
+++ b/engine-mr/src/test/resources/multi_file_str/data_4
@@ -0,0 +1,2041 @@
+003511e3-3c9f-47b8-a4f1-59a982d41780
+00364074-2efe-4dcc-a526-06c03a9bd64e
+003d3973-4b74-43e4-862d-3c93423c9016
+005b851f-6f5e-4bce-b44f-ab901bb19266
+00ae59d3-5f1a-4ffa-a9dd-ace8b1b616b3
+00e1077e-5db6-4739-b728-dcfdebbeac89
+00fd44f7-ef50-48a6-952f-83758e8b73fc
+01412f09-1d12-479d-82a5-a0cc4cd3eb1d
+01439f30-8956-47a6-afdc-1db126990f68
+01573fad-0365-4162-b37f-363f05212724
+0159fef7-4ca4-492f-8595-679dc92c8d96
+01647025-c8a8-4e2f-929f-2a7be01a3d2f
+01a95d04-edfa-4992-abbe-a4effb60608b
+01f38832-7270-4732-9321-7b36ab453241
+01f6e7fe-db14-4caf-b459-114eb059e6ef
+01f95d79-5998-4f22-85ed-ab120baeccc4
+0213b384-5f87-450a-b7b0-7d28875567d3
+021657d7-7ed4-44b4-b310-2bcc030c5f17
+02320923-2831-433d-9936-b35532a74935
+0285c5a8-5391-4147-926d-0e8964613bc9
+02b9a18b-ded9-44f0-ba6d-d3276866b8de
+02c23499-80c0-4821-a60f-ca446640018d
+03196211-662e-40f7-9534-33b4984448e0
+03332ef9-4953-4b34-9d90-bb6418d599c7
+033ca26a-2001-4393-b450-20aedc8b7d73
+03896fe0-7e8b-4901-adb7-2d967bab0c32
+038ede24-4288-437c-ab82-8b3ecd8bf0a0
+03962dbf-26f6-450c-8d84-4506eb00b935
+03a4b0f3-a9ad-4780-93cc-cbb5fb0338bd
+03c3576d-86d7-4267-830f-445dc6cf47aa
+03c9a74a-bcb2-4d4a-8019-6da293cf90b9
+03cf2884-62fa-4ea7-86d2-76bb9a673927
+03dee29f-e388-4576-8943-deb3c3b80c83
+040047b2-c609-4677-8de2-17621ab043d6
+042c1afa-26e5-4d8a-8256-7402db698b2e
+0464ad68-9c3e-4e97-82aa-2e9e722b06f7
+046714c9-01ee-44d7-83d5-07bcea2d7ba2
+048f02ba-1183-4290-8114-bf5abb74324f
+0502e1b7-7cd6-4d8f-be64-9791f01c05a9
+0507561e-8fe2-488c-98e5-319d85005b36
+053ca4ca-f03f-4715-bf3c-d3fd81c90250
+054232b8-aba9-43d7-822e-ccd65ad2d8e2
+054647df-c970-48b0-a73b-7d9d779d6f75
+054e6f2b-e0e4-4d9c-a9a3-b7262b9ca776
+0556b4b5-5081-4c3b-ac4f-a6301aa87121
+05626b77-5944-4389-b398-92b80a4599cb
+059fedcc-024d-436b-9d75-b71a28aa934f
+05a5075a-5b22-4a8e-b0ab-dc496eaa097e
+05c7a17e-5842-4ea3-ae5f-1f5b1a54bd5a
+05e24866-3424-4f4d-b703-409c4dbf8104
+05f77326-e88e-4785-9c7f-c7f9c3df6652
+0625a0df-eea3-4f8b-bba2-4c8d3c5c9351
+062c94fc-3a06-4f89-9818-f61897482570
+06310498-3813-4554-ae61-57bea69db0b1
+0653727a-3fad-4c2f-9b89-a9982854a9c3
+068fbb79-03dd-49cb-92ce-1f65049c4543
+06b2785c-ec04-4f46-87ad-f76e68422ae0
+0710dad8-485e-4c3e-b0fe-2cd569d90507
+071d54d9-f884-4b0c-a8f4-85bc436475e1
+0725acc2-5ae7-4ddb-af1a-90e5c98f33c2
+072ba507-beca-4680-94a2-ceb062cb1890
+0748f60e-0c63-4e99-b804-ab872a09aef7
+0768b6d9-2dd2-41a7-9193-42712a34359b
+078966b8-26cd-4416-80d5-754616122c3c
+07adc055-aace-40c6-a7da-ebac55d46e91
+0811fff3-186f-4637-ae9d-5740fef969c7
+082781e5-fa2e-443a-8ba2-ec789423eaf3
+08327101-0aa3-48ee-b25d-6d787ff144e7
+085310c6-577e-41a2-9214-f63f25fc1fde
+08624ffe-fe03-4e4f-b39a-c9d83b847a3b
+0877cc24-e51f-48b7-969d-43dad21289c7
+0892766e-1c4a-4c80-b211-8967f693eddc
+08cc45ba-12e1-496c-9a72-78c401fe58d4
+08dfa8fc-c109-4322-a933-cb01119ff572
+08e883e1-fdc3-481b-ac4d-68a715b46a5b
+08f7f315-6a9b-4575-b613-593e5f04cd7b
+095436d6-8a5e-4b90-ac87-f2945ac673eb
+096c8285-50ac-4373-a0ee-3c30aac7164e
+09967e6a-343a-4629-bfb5-82413379839b
+09a6886b-a3e0-43d1-b2a4-7d090b171f4d
+0a060fcb-a1bd-40cc-8e5e-99c83287f22b
+0a15d2b3-7c89-4113-a6cc-fbc6e136030e
+0a445c63-85a3-42e0-bcad-aa341e4b0ea1
+0a5bd490-a8ad-4a04-bb36-bc722ff97f6d
+0a6c64fe-6ac2-40dc-857f-90a04474c92b
+0a8d5267-ef87-4223-b003-01ee7f716926
+0a8dfaec-1c57-4baf-aeb9-84c6763c603b
+0a9a149c-3704-4ac5-90de-ce06aaf883ed
+0a9aedb3-e4f1-4ab9-82bf-52f7b3c3d66c
+0aa52441-b644-4ca7-9f6d-cfb314a255ab
+0acffdf7-12bc-41c3-a012-90ecb3306d51
+0b3b9628-ad13-42d3-9f9e-2a45b8280708
+0b65e109-8018-4054-93da-d860f94ba860
+0b9fb904-a1c1-4ca4-af80-30e9e1d6849a
+0ba9bb17-ab1a-451d-8485-fd09aab2ff56
+0bc79581-b054-49b3-a98f-f12170516631
+0bd84756-af77-44e3-ad3d-28190a916fa5
+0c20ec9b-24f7-4436-b7a9-22c88dcbcadb
+0c3edb98-1162-4824-add7-3d3f57e811c2
+0c523875-3b57-4ba2-b3d1-c916c4077f1e
+0c56bb2c-c018-4901-9a3c-0c191cc83d2d
+0c5dba85-62f0-47f2-a2a1-04d075c22fea
+0c8f0ca1-c8c5-4f56-bde5-0980cb6b4663
+0cab8877-0532-43b2-b5ae-9077280e960d
+0cbf8745-20a2-4180-a032-5513f3ae7102
+0d6d435d-fcb2-41c7-adfb-70e8ee2a2ca4
+0d7a5055-3371-42bc-88e7-d0b91b68893c
+0d9d827c-98a4-4c28-913e-30913a2f896c
+0dde2519-541e-45bd-887d-441d7460ed2a
+0ddee6a9-34df-4177-bc06-5127326cb6e9
+0e05c1d8-eb2b-4722-82f9-359f5bd19b37
+0e285454-71a5-40ef-bb5c-bf32a32dc3a4
+0e73e1e5-f753-494f-9d6d-7caef066f029
+0e7e6f43-2c71-428c-8ff9-39a1b90cf0e5
+0e935a16-8784-43f6-b7a1-f59e3cd00f0c
+0eb3fa79-3b93-455d-a25a-d224a9e55f0a
+0eb7bd99-97b8-41c2-9d8b-371a3bf40161
+0ed3a37f-5e6e-450c-82b1-75d6368f8bd3

[10/10] kylin git commit: KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some useless MR files.

2016-11-22 Thread liyang

KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some 
useless MR files.

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b7fe610
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b7fe610
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b7fe610

Branch: refs/heads/master
Commit: 2b7fe6106d5363d9993dfd7b7f27c49319ed7d44
Parents: 79cb59c
Author: xiefan46 <958034...@qq.com>
Authored: Thu Nov 17 14:21:38 2016 +0800
Committer: Li Yang 
Committed: Tue Nov 22 18:39:35 2016 +0800

--
 .../kylin/cube/AggregationGroupRuleTest.java|2 +-
 .../validation/rule/DictionaryRuleTest.java |1 +
 .../apache/kylin/dict/DictionaryGenerator.java  |4 +-
 .../apache/kylin/dict/DictionaryManager.java|3 +
 .../dict/NumberDictionaryForestBuilder.java |5 +
 .../dict/TableColumnValueSortedEnumerator.java  |  154 ++
 .../kylin/dict/TrieDictionaryForestBuilder.java |2 +-
 .../apache/kylin/dict/lookup/SnapshotTable.java |2 -
 .../kylin/dict/TrieDictionaryForestTest.java|8 +-
 .../org/apache/kylin/source/ReadableTable.java  |   17 +-
 .../apache/kylin/engine/mr/DFSFileTable.java|   36 +
 .../engine/mr/DFSSingleFileTableReader.java |  218 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |4 +-
 .../apache/kylin/engine/mr/SortedColumn.java|  100 +
 .../kylin/engine/mr/SortedColumnReader.java |  136 ++
 .../engine/mr/steps/CreateDictionaryJob.java|5 +-
 .../mr/steps/FactDistinctColumnPartitioner.java |5 +-
 .../mr/steps/FactDistinctColumnsCombiner.java   |4 +-
 .../engine/mr/steps/FactDistinctColumnsJob.java |2 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java |3 +-
 .../mr/steps/FactDistinctColumnsReducer.java|6 +-
 .../mr/steps/FactDistinctHiveColumnsMapper.java |   22 +-
 .../engine/mr/steps/SelfDefineSortableKey.java  |  128 ++
 .../apache/kylin/engine/mr/steps/TypeFlag.java  |   28 +
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |5 +-
 .../fdc2/FactDistinctColumnPartitioner2.java|   47 -
 .../fdc2/FactDistinctColumnsCombiner2.java  |   45 -
 .../mr/steps/fdc2/FactDistinctColumnsJob2.java  |  148 --
 .../fdc2/FactDistinctColumnsMapperBase2.java|  103 -
 .../steps/fdc2/FactDistinctColumnsReducer2.java |  254 ---
 .../fdc2/FactDistinctHiveColumnsMapper2.java|  232 --
 .../mr/steps/fdc2/SelfDefineSortableKey.java|  131 --
 .../kylin/engine/mr/steps/fdc2/TypeFlag.java|   28 -
 .../kylin/engine/mr/SortedColumnReaderTest.java |  312 +++
 .../mr/steps/NumberDictionaryForestTest.java|   17 +-
 .../mr/steps/SelfDefineSortableKeyTest.java |2 -
 .../src/test/resources/multi_file_double/data_0 | 2006 +
 .../src/test/resources/multi_file_double/data_1 | 2093 ++
 .../src/test/resources/multi_file_double/data_2 | 2006 +
 .../src/test/resources/multi_file_double/data_3 | 1947 
 .../src/test/resources/multi_file_double/data_4 | 1948 
 .../resources/multi_file_empty_file/empty.txt   |0
 .../multi_file_empty_file/has_value.txt |2 +
 .../src/test/resources/multi_file_int/data_0| 1987 +
 .../src/test/resources/multi_file_int/data_1| 1989 +
 .../src/test/resources/multi_file_int/data_2| 1998 +
 .../src/test/resources/multi_file_int/data_3| 1996 +
 .../src/test/resources/multi_file_int/data_4| 2030 +
 .../src/test/resources/multi_file_str/data_0| 2029 +
 .../src/test/resources/multi_file_str/data_1| 1990 +
 .../src/test/resources/multi_file_str/data_2| 1992 +
 .../src/test/resources/multi_file_str/data_3| 1948 
 .../src/test/resources/multi_file_str/data_4| 2041 +
 .../org/apache/kylin/source/hive/HiveTable.java |3 +
 54 files changed, 31200 insertions(+), 1024 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2b7fe610/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 1c43093..1950b02 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -38,7 +38,7 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-public class AggregationGroupRuleTest extends LocalFileMetadataTestCase{
+public class

[02/10] kylin git commit: KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some useless MR files.

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b7fe610/engine-mr/src/test/resources/multi_file_str/data_3
--
diff --git a/engine-mr/src/test/resources/multi_file_str/data_3 
b/engine-mr/src/test/resources/multi_file_str/data_3
new file mode 100644
index 000..f8195c6
--- /dev/null
+++ b/engine-mr/src/test/resources/multi_file_str/data_3
@@ -0,0 +1,1948 @@
+0096b54c-a8ba-4c45-b579-14d6dbe135dd
+00980ae3-dbcc-45a2-b003-6f761a99d32d
+0125a3fe-53a4-413f-9c79-1a221e5821ca
+0125d522-8c77-4dfc-8b48-2e3e693955c7
+0125f5c9-fd20-41c8-bf8b-24929664ce4e
+0133fa28-78cc-4b70-a63c-51cf77e0fe70
+0165b916-74fd-4d5a-a26e-56828bde147b
+016d1cfb-129b-4788-893e-a124034ebdfb
+02237af5-4665-48b7-ad55-25db423a3d81
+02318922-1d3e-49fd-9d1c-12b9764ae9f7
+0232a118-7b48-4b0b-8e16-0dfd0633e236
+02576c08-1082-47f4-9efb-0dcd12698d93
+027768b7-7775-4ae5-9ec5-9a68d601f615
+029329b5-44b5-4127-bb39-c7903b5a9804
+02af5639-679c-4694-81d1-a8d61617246c
+02ca7081-41b4-4ea7-9fc4-a921542bb829
+02e23288-21ef-4c3a-8370-83a3735b523d
+02e72bcc-48b5-4c52-8fa4-22d026290b48
+02f266bc-bb2d-476d-b8c2-69efa25de39d
+032825c3-3660-4b1d-8b61-64afa8a5a0ce
+03287e77-ccd7-43cb-a81e-5146905e558d
+032b09d7-c50a-48a7-bc02-ff98794f1303
+0338e66e-1f09-4da3-8846-2995b90ed4f2
+034677fa-5dd5-4a2d-a25a-e3130dff3ae0
+03926636-7251-488e-811d-670aaabe0b15
+039cc454-99f8-4081-9e04-58142af8b791
+03ccf53c-851b-4eb6-96dd-ee32a4349ce7
+03cdf1ae-947d-4dac-80a5-542bf633de3c
+03ece7f8-912d-4b7c-9fb8-d8bce8b3c03e
+03ef4a93-7ecb-411f-bf28-91eb4f4d5390
+04270d3d-cc3e-40ef-a95a-32ee4b60080f
+04682d83-3f08-4daa-8f4b-4377439d7e67
+04836cef-a499-45e9-b4db-e5cbe6a47360
+0499503b-7ffc-4d2f-91c6-083f0437ce25
+04aa3c7a-4d78-46ad-abbd-3a39d5aa1fd7
+04d64d9b-d56b-44f8-9165-c67c95b269b4
+04db6ade-4ddb-40ed-a64c-086b69afb54d
+04e4e1df-06d7-4830-a6ca-b5aa8bcc3bae
+04fa63c4-26a3-41c4-bec9-80af3e3f9054
+05122c37-3ab2-4995-9879-0dc60a847acd
+0523a8a8-49c1-477a-9bb8-84604a4585e1
+05460f94-1a47-4857-9935-c6dade2772f9
+057cc7c2-c290-4d74-ad1b-4c088a52f9f0
+05910d98-139c-475c-bff3-deec71f149e0
+05960493-10c8-4cbc-8e33-1d26b4094db9
+05b0669a-77c2-4d94-bf47-03de9d8b2753
+05bc4a47-caa2-4c33-a767-dfe919db0922
+05cc3e00-2d7a-4dc2-aad2-748a3f873e3f
+060c973d-53b2-40ee-9a56-ae3c213584cd
+0612b50c-e374-4735-8860-acdfe633c575
+06167be2-490f-440a-b56d-b67c8a3cfb4c
+061b7f3e-d1e1-481a-85c3-e2925eb33299
+065288c8-9f18-46d7-8574-268f5faa2153
+065a5e40-d5c2-4a0d-bd17-f2813f3fae41
+06b50675-1c2f-445d-a150-59216dafda8b
+06c11a51-82eb-4b3e-9e5b-6513380a992c
+06c9051d-800c-4aea-8215-4ed50dd2134c
+072ed7d6-edd7-4f9d-bda5-741faa78eb14
+077367d1-9503-4c05-ac4e-4ca6d625dd76
+07a3605b-42ad-4713-ad68-66d82791d913
+07aa0f29-332f-4d8e-bf16-464b13d0823b
+07fd8163-fcbc-49bb-8e36-c67ada2f7995
+08051883-1810-4139-be4b-c35243ff97c7
+08053d2e-2097-4211-9b94-3815d69935d8
+08085995-1ca5-4bab-8e48-1d85d4fc7997
+080d85cd-505d-4041-96a4-929fdb6e67f6
+087fd627-9ce4-4b85-9c44-e07aab81806c
+0899d820-1d47-4d4e-94b8-9a09feaaa9f4
+08abd59e-e720-4ea0-81a4-c40b4d98cc2e
+08ce11e7-ea27-4df0-90f0-d8dc429b7b06
+08e6b700-d15c-4f92-a9dd-2930135af5e4
+0905ff6b-a93d-45d0-999f-369cbb601d90
+090fcffb-842b-4c5d-a119-3af601b121a8
+0917adbf-036a-471f-947d-a3ea53c9f5c9
+093849cb-654e-4cfe-ad5e-d6407d597e98
+0946b9c7-7adf-4587-b514-ced7816d6530
+09677dc3-75c9-4349-92d5-6d29a77ef923
+098ba031-8626-4ebe-9abf-0817d0282a6a
+0995a065-7917-4f0e-b28b-0ddfde40b7b8
+09a708c9-28a5-48dc-9b0c-0abe233b24d4
+09dfbd16-ed86-4463-9317-ddd2f7bff51e
+09f3cd27-b43d-476e-a2d0-8ae3e52e85ba
+09f7c9c6-6b17-460b-a3b9-598017533c08
+0a13e299-faa4-443c-b091-9664d7e9987d
+0a349398-b03d-43df-83ee-cd4817cd55fd
+0a4edb8c-acd4-43aa-961f-15fee1b7eaaa
+0aae781d-c933-4f21-9b28-ee4e1a067010
+0b1aece5-6f38-429a-9364-177d330ebeca
+0b5fed21-ce2c-4d9e-96a3-e1ba98cf05d6
+0b6f398e-187a-432f-845e-b6492f4d5786
+0b9e17e7-3e27-4050-ab69-b3c43b1d1e21
+0c10c194-cedf-48d9-b657-578a588894f3
+0c13bfe1-7696-4581-8a31-b8d26af3ef0c
+0c418c78-4164-423d-ad3e-52d0d12f4e51
+0c8bcc53-304c-47af-a508-64304085d2d3
+0cb9ce1d-b7d1-47de-8b7d-ae5db873fe01
+0ce24a01-3e71-4313-8db8-93b3a0154861
+0cfbe1c7-f421-4bdd-bfec-7e911734b120
+0d49500e-d916-4acb-94f4-39f24631ba32
+0d56c741-99fe-46c3-ac08-94d4ea181bea
+0d99bc6e-de2f-42d6-ae2b-94a513653c18
+0d9c1820-fec9-4417-bef8-05dfe93e6ace
+0db6ce11-13f8-4115-bf53-a427ab9873b5
+0dba3aa5-8cbf-4e5b-a2f7-a89a64e2a0e9
+0dc6e590-1032-4595-a4fc-03e39f124061
+0e1c80dd-329b-49e9-bf59-cc8c088bd1ad
+0e3cea58-74d6-4924-ac8d-aa05ad28098d
+0e671f4d-2b9a-411f-94c5-26af38be1abd
+0e76d428-b0f5-4cea-ad19-41f87bbdafe1
+0e97a938-c8e4-4ae7-8c44-b2c9144642d3
+0eb3e881-951a-490c-922d-0f58b1180467
+0edf9bce-c6ee-4c28-bb11-dd2731dd4090
+0eff50b6-21dc-42db-a760-52ed81a53883
+0f197e9e-c1be-4892-821b-0466ea21b1f6
+0f203bf6-e0d2-4b96-a10c-bd60639d6e44
+0f43abde-8225-4f8f-a617-4c3a08d794f3
+0f9b0c36-fa01-41c8-86aa-fb7bd09a58b4
+0fcd9c32-8c07-4000-a007-42931b799d70
+0fe2fe4e-59c7-42b7-96b1-c9dd9dc9e74f
+10121383-3843-4f29-9ac3-63d51855d297

[08/10] kylin git commit: KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some useless MR files.

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b7fe610/engine-mr/src/test/resources/multi_file_double/data_1
--
diff --git a/engine-mr/src/test/resources/multi_file_double/data_1 
b/engine-mr/src/test/resources/multi_file_double/data_1
new file mode 100644
index 000..f3c6d42
--- /dev/null
+++ b/engine-mr/src/test/resources/multi_file_double/data_1
@@ -0,0 +1,2093 @@
+1.56
+9.356
+13.512
+14.032
+17.67
+19.7599987
+20.786
+22.3599985
+34.31
+35.36
+39.5200024
+45.240006
+52.1
+57.200013
+61.3600156
+62.9200165
+65.520017
+67.080015
+67.600015
+70.720013
+71.240012
+75.920009
+83.720003
+84.240002
+90.479998
+91.519997
+97.759992
+98.71
+105.039986
+108.159984
+109.199983
+111.279982
+112.319981
+112.83998
+114.399979
+115.959978
+116.479978
+121.159974
+122.719973
+124.799971
+125.319971
+127.39997
+130.519973
+133.119978
+133.63998
+134.67998
+137.799987
+138.83999
+140.919993
+141.959995
+144.04
+145.08
+146.640004
+147.160005
+148.27
+148.720008
+151.320014
+152.360016
+154.44002
+156.520024
+157.040025
+160.680032
+162.760036
+165.880042
+167.440045
+168.480047
+169.52005
+171.600053
+172.120054
+173.160057
+174.72006
+177.320065
+180.960072
+182.520075
+183.040076
+187.200084
+187.720085
+188.760087
+189.280088
+190.32009
+192.920095
+195.0001
+200.72011
+203.320116
+205.92012
+208.000125
+210.08013
+210.60013
+211.640132
+212.680134
+215.80014
+216.320141
+216.840142
+217.360143
+219.440148
+221.520152
+228.280165
+230.36017
+231.920172
+233.480175
+234.000176
+235.040178
+237.640183
+238.680185
+240.240188
+241.800192
+242.320193
+243.360195
+245.9602
+248.560205
+249.080206
+250.64021
+251.68021
+257.400214
+257.92021
+258.96021
+265.720185
+267.28018
+275.08015
+283.40012
+288.080106
+288.600105
+290.6801
+292.24009
+294.320085
+299.7
+303.160054
+310.44003
+312.2
+313.04002
+321.87999
+323.43998
+324.47998
+332.79995
+333.839946
+334.87994
+335.91994
+336.43994
+342.15992
+343.71991
+347.3599
+352.55988
+353.07988
+354.639874
+355.67987
+356.719866
+359.839855
+363.47984
+363.99984
+370.23982
+373.35981
+374.399804
+375.9598
+381.15978
+381.67978
+382.19978
+387.91976
+388.439755
+389.47975
+390.51975
+392.07974
+394.679734
+395.71973
+397.279724
+398.31972
+398.83972
+404.0397
+407.15969
+409.75968
+414.95966
+417.039655
+418.07965
+418.59965
+420.159644
+426.91962
+430.03961
+430.55961
+435.75959
+436.799586
+437.83958
+438.87958
+439.919575
+441.47957
+445.639555
+455.51952
+456.559517
+458.11951
+458.63951
+460.199504
+465.399486
+469.55947
+473.719457
+483.59942
+486.199413
+487.75941
+488.799404
+491.399395
+496.07938
+497.639373
+498.67937
+505.439346
+509.59933
+511.679324
+512.19932
+518.9593
+521.03929
+521.55929
+525.19928
+526.75927
+528.31927
+536.11924
+539.75923
+548.5992
+550.67919
+559.51916
+562.11915
+563.67914
+565.75914
+567.31913
+567.83913
+570.43912
+573.55911
+574.5991
+575.1191
+576.6791
+583.95907
+587.59906
+588.63906
+593.83904
+597.47902
+601.11901
+601.63901
+604.759
+606.83899
+607.87899
+609.43898
+616.71896

[07/10] kylin git commit: KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some useless MR files.

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b7fe610/engine-mr/src/test/resources/multi_file_double/data_3
--
diff --git a/engine-mr/src/test/resources/multi_file_double/data_3 
b/engine-mr/src/test/resources/multi_file_double/data_3
new file mode 100644
index 000..816bd32
--- /dev/null
+++ b/engine-mr/src/test/resources/multi_file_double/data_3
@@ -0,0 +1,1947 @@
+0.52
+2.08
+4.16
+12.993
+15.071
+19.2399988
+23.384
+23.9199984
+26.5199982
+27.039998
+28.079998
+28.58
+29.639998
+31.178
+34.836
+36.406
+37.440001
+42.120004
+43.680005
+45.760006
+46.2800065
+47.8400074
+49.920009
+52.52001
+56.680013
+62.400016
+63.960017
+66.560016
+74.88001
+77.480008
+79.040006
+81.640004
+89.439998
+93.55
+94.119995
+95.159994
+96.13
+98.279992
+102.439988
+109.719983
+110.239982
+111.799981
+118.039976
+118.559976
+119.079976
+121.679974
+125.83997
+127.919969
+131.559975
+132.599977
+141.439994
+143.519998
+146.120003
+147.680006
+149.24001
+150.800013
+155.480022
+158.600028
+159.12003
+162.240035
+164.84004
+166.400043
+167.960046
+169.48
+174.200059
+176.280063
+182.74
+189.80009
+191.360092
+193.960097
+198.120106
+205.40012
+207.480124
+208.520126
+211.12013
+212.160133
+214.760138
+222.560154
+223.600156
+225.68016
+231.40017
+232.440173
+234.520177
+235.56018
+245.4402
+253.240214
+256.880216
+261.0402
+263.120194
+264.16019
+265.200186
+267.80018
+268.320176
+268.840174
+273.00016
+274.040156
+279.24014
+279.760136
+281.84013
+282.360126
+282.880125
+283.92012
+284.96012
+289.1201
+293.28009
+295.36008
+296.920075
+298.48007
+299.520066
+305.240046
+307.84004
+308.360035
+316.680006
+319.75
+320.83999
+324.8
+325.519975
+328.119966
+328.639965
+329.15996
+337.3
+340.59992
+348.919894
+350.999886
+351.519884
+352.03988
+359.31986
+362.439846
+365.03984
+365.559835
+367.63983
+368.159826
+369.19982
+370.75982
+371.279815
+372.31981
+372.83981
+374.9198
+380.119784
+383.239774
+383.75977
+384.27977
+384.79977
+386.87976
+388.959754
+399.879715
+400.399714
+401.43971
+402.479706
+404.5597
+405.079697
+407.67969
+408.199686
+410.799677
+413.39967
+414.439664
+423.79963
+425.879624
+432.6396
+436.27959
+439.399577
+440.439573
+442.519566
+445.119557
+447.19955
+450.31954
+452.39953
+452.91953
+457.079515
+459.679506
+462.279497
+465.919484
+466.95948
+469.039473
+474.759453
+475.79945
+476.839446
+477.359444
+478.39944
+479.959435
+484.11942
+485.679415
+486.71941
+489.3194
+490.3594
+491.919393
+492.43939
+493.47939
+496.599377
+499.719366
+502.839355
+503.359353
+503.87935
+504.39935
+504.91935
+507.51934
+510.63933
+525.71928
+527.79927
+531.43926
+535.59924
+542.35922
+544.95921
+549.63919
+551.71918
+552.23918
+554.31918
+555.87917
+556.91917
+558.47916
+560.03916
+560.55915
+569.91912
+573.03911
+582.39908
+584.47907
+584.99907
+586.03906
+586.55906
+588.11906
+589.67905
+590.71905
+597.99902
+600.07902
+605.799
+610.47898
+610.99898
+613.07897
+619.31895
+619.83895
+621.39894
+624.51893
+627.11892
+629.19891
+631.27891
+631.7989
+634.91889

[06/10] kylin git commit: KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some useless MR files.

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b7fe610/engine-mr/src/test/resources/multi_file_int/data_1
--
diff --git a/engine-mr/src/test/resources/multi_file_int/data_1 
b/engine-mr/src/test/resources/multi_file_int/data_1
new file mode 100644
index 000..a728f10
--- /dev/null
+++ b/engine-mr/src/test/resources/multi_file_int/data_1
@@ -0,0 +1,1989 @@
+1
+4
+6
+17
+18
+23
+33
+34
+46
+54
+65
+81
+91
+98
+99
+101
+105
+106
+107
+109
+117
+120
+123
+134
+135
+138
+141
+142
+163
+174
+175
+176
+178
+181
+183
+187
+188
+207
+228
+239
+256
+257
+261
+263
+268
+270
+272
+281
+282
+288
+289
+290
+291
+303
+312
+313
+320
+324
+338
+355
+357
+363
+368
+369
+371
+376
+383
+392
+399
+401
+406
+409
+411
+419
+431
+434
+438
+440
+441
+449
+450
+464
+468
+475
+476
+478
+481
+488
+489
+495
+501
+519
+521
+522
+527
+529
+531
+535
+544
+552
+562
+567
+572
+574
+576
+585
+588
+592
+595
+596
+601
+604
+611
+617
+620
+624
+628
+631
+632
+635
+636
+640
+641
+644
+646
+656
+657
+661
+673
+676
+686
+693
+694
+700
+702
+704
+709
+711
+716
+718
+728
+733
+734
+737
+744
+748
+756
+760
+762
+764
+773
+776
+784
+786
+789
+790
+800
+804
+808
+815
+816
+818
+819
+820
+822
+823
+827
+839
+845
+846
+847
+848
+851
+862
+863
+867
+868
+879
+891
+899
+902
+907
+913
+916
+917
+918
+921
+924
+925
+928
+931
+936
+942
+943
+946
+949
+953
+955
+956
+959
+962
+968
+981
+986
+987
+989
+998
+999
+1002
+1006
+1009
+1020
+1028
+1031
+1033
+1035
+1036
+1037
+1046
+1055
+1056
+1068
+1078
+1080
+1082
+1086
+1087
+1089
+1090
+1091
+1099
+1100
+1108
+1113
+1116
+1121
+1124
+1125
+1127
+1129
+1138
+1144
+1148
+1156
+1162
+1173
+1176
+1180
+1182
+1189
+1191
+1192
+1194
+1213
+1214
+1217
+1233
+1234
+1237
+1239
+1242
+1245
+1246
+1252
+1258
+1259
+1262
+1272
+1276
+1277
+1278
+1279
+1297
+1303
+1312
+1313
+1323
+1341
+1342
+1344
+1355
+1359
+1362
+1376
+1377
+1379
+1387
+1392
+1406
+1409
+1410
+1422
+1423
+1428
+1430
+1431
+1436
+1437
+1445
+1451
+1454
+1472
+1474
+1477
+1478
+1494
+1495
+1501
+1506
+1507
+1512
+1514
+1527
+1530
+1531
+1533
+1536
+1537
+1538
+1542
+1545
+1551
+1556
+1559
+1565
+1572
+1577
+1588
+1595
+1599
+1607
+1616
+1621
+1623
+1636
+1661
+1662
+1667
+1671
+1672
+1678
+1681
+1682
+1684
+1693
+1705
+1716
+1717
+1721
+1725
+1733
+1739
+1758
+1759
+1762
+1768
+1774
+1782
+1783
+1785
+1798
+1802
+1804
+1819
+1823
+1826
+1830
+1832
+1844
+1847
+1851
+1853
+1865
+1866
+1869
+1870
+1873
+1880
+1889
+1890
+1897
+1903
+1906
+1909
+1910
+1919
+1922
+1926
+1931
+1934
+1937
+1938
+1942
+1951
+1954
+1956
+1967
+1969
+1974
+1983
+1990
+2008
+2012
+2018
+2020
+2035
+2037
+2042
+2043
+2053
+2054
+2057
+2062
+2069
+2071
+2073
+2076
+2077
+2082
+2089
+2096
+2099
+2101
+2102
+2104
+2110
+2113
+2118
+2119
+2120
+2124
+2125
+2128
+2143
+2145
+2152
+2161
+2169
+2171
+2173
+2185
+2191
+2192
+2195
+2198
+2200
+2205
+2214
+2233
+2240
+2252
+2258
+2274
+2279
+2282
+2283
+2289
+2297
+2301
+2302
+2304
+2305
+2310
+2312
+2321
+2329
+2331
+2347
+2348
+2352
+2355
+2360
+2362
+2370
+2371
+2372
+2374
+2380
+2381
+2383
+2399
+2406
+2409
+2411
+2423
+2424
+2429
+2436
+2440
+2443
+2444
+2446
+2448
+2449
+2452
+2458
+2462
+2465
+2467
+2477
+2486
+2488
+2489
+2493
+2501
+2511
+2515
+2524
+2526
+2536
+2540
+2545
+2549
+2553
+2554
+2555
+2556
+2563
+2564
+2567
+2568
+2570
+2573
+2576
+2579
+2598
+2611
+2616
+2617
+2620
+2637
+2640
+2647
+2648
+2650
+2651
+2660
+2670
+2674
+2676
+2677
+2678
+2686
+2689
+2697
+2701
+2703
+2704
+2709
+2715
+2720
+2726
+2728
+2732
+2739
+2746
+2748
+2758
+2760
+2761
+2764
+2767
+2772
+2773
+2782
+2784
+2786
+2803
+2807
+2811
+2817
+2826
+2834
+2837
+2857
+2861
+2862
+2878
+2881
+2884
+2889
+2903
+2911
+2912
+2917
+2919
+2920
+2921
+2935
+2942
+2945
+2947
+2948
+2953
+2965
+2966
+2976
+2979
+2982
+2983
+2989
+2990
+2992
+2995
+2996
+3005
+3008
+3009
+3010
+3011
+3020
+3022
+3023
+3024
+3025
+3034
+3035
+3038
+3045
+3047
+3059
+3073
+3078
+3079
+3083
+3090
+3091
+3092
+3094
+3098
+3100
+3103
+3109
+3112
+3115
+3116
+3119
+3120
+3124
+3128
+3130
+3131
+3141
+3146
+3150
+3153
+3158
+3163
+3165
+3169
+3184
+3187
+3192
+3195
+3196
+3197
+3213
+3220
+3226
+3233
+3243
+3248
+3256
+3260
+3272
+3276
+3280
+3287
+3288
+3290
+3294
+3313
+3318
+3327
+3332
+
+3334
+3335
+3347
+3352
+3359
+3368
+3370
+3372
+3373
+3382
+3388
+3393
+3395
+3401
+3411
+3412
+3417
+3419
+3420
+3428
+3432
+3444
+3446
+3448
+3461
+3462
+3467
+3470
+3471
+3472
+3481
+3492
+3501
+3502
+3508
+3513
+3514
+3519
+3522
+3525
+3527
+3528
+3535
+3536
+3545
+3566
+3578
+3596
+3601
+3610
+3642
+3643
+3645
+3651
+3652
+3657
+3661
+3666
+3672
+3674
+3675
+3677
+3686
+3687
+3689
+3693
+3700
+3715
+3716
+3717
+3718
+3738
+3739
+3744
+3745
+3746
+3747
+3755
+3757
+3766
+3769
+3770
+3774
+3776
+3777
+3780
+3782
+3784
+3786
+3800
+3805
+3806
+3808
+3814
+3824
+3829
+3833
+3836
+3838
+3855
+3857
+3860
+3868
+3880
+3883
+3896
+3898
+3912
+3926
+3927
+3929
+3938
+3942
+3945
+3946
+3947
+3953
+3954
+3969
+3970
+3977
+3982

[03/10] kylin git commit: KYLIN-2202 fix the conflict between KYLIN-1851 and KYLIN-2202. Remove some useless MR files.

2016-11-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b7fe610/engine-mr/src/test/resources/multi_file_str/data_2
--
diff --git a/engine-mr/src/test/resources/multi_file_str/data_2 
b/engine-mr/src/test/resources/multi_file_str/data_2
new file mode 100644
index 000..affa581
--- /dev/null
+++ b/engine-mr/src/test/resources/multi_file_str/data_2
@@ -0,0 +1,1992 @@
+0045ee27-7fc4-4b1c-9ed6-8902e80692c9
+00565369-b833-4e1f-a8ef-6f825242a4d4
+00795204-5a38-4af5-8375-bba5e4382dc7
+00b25c18-9fd6-4b90-892c-a57ffd5841b3
+01401e6f-8dc0-43a3-99ad-6bc1c678bd21
+015c6973-3f32-4a0a-88f2-1f82a1b9f6a2
+01654b43-2c4b-4af1-96b4-fdd8a2b75536
+016da6e6-f4c2-48c6-adf7-4aaf6a7fd850
+016db1b7-82f8-438c-9aaf-125ed70f7975
+016fda45-f76d-41b2-b3f3-d693f38141e9
+0178b2fc-907e-4867-80fc-a2c840978049
+01bb4af2-f5a8-4d6f-8a69-08a5ea3601f7
+01db1b24-cde1-450e-a979-27940c524631
+02078814-e580-4ae7-b77b-7a320fe28282
+0213ef6b-ba04-4140-b5d5-536bf156c226
+02185e06-c1db-4402-8057-e54a3755cf68
+0228d81b-3798-4110-8448-35efb404e46e
+023a4071-3e92-4918-8f06-f213b9bc3b4e
+025c8eb6-dc5e-47c5-a921-ef231a71c6c3
+028369d7-d3a0-4ec2-b84d-517ea9f1bdcb
+029d97cb-86d1-46fe-ac71-e0774ef20c6e
+02e8e373-4ac8-4dcd-b798-8520a641b080
+02f2775c-fb9e-4694-9a22-2642d6d8244b
+03135972-ec17-447b-8042-fd2eb1ee4260
+033896e7-41e8-44f0-b94e-26a0370d1ee0
+038362fe-1152-4747-a143-dc1557168bae
+0397a221-5c5e-41f6-a972-cf360394d758
+039d1f7d-37f2-4864-8ea4-fb4dee9ef071
+03c6a446-4e75-466d-82b1-1b584ebf50c9
+03fb2a98-60da-429d-9588-a37ddad214ba
+040b9fd9-306a-413d-9924-9aeacfd39afc
+040c3f99-f755-4fcb-b537-15a51d73d29d
+040ce641-8ebc-4364-a8c2-bacabc96443d
+046016f3-6d75-4174-b9f9-5e067aadff7d
+04ad7bb4-650d-4d01-930d-22ebad831bf1
+04d0ed08-3bee-4554-90de-0e8883962c3e
+04e6bcdb-ac32-49ce-a82e-3763aee08b5c
+04e90c02-fe9d-4530-8012-7dcd6147cce6
+05336a1c-c0c5-4d11-be62-95da26541369
+0550854c-43d8-47a8-a9f4-7043de9ed2b4
+05663e89-0efd-47fa-9a78-4804dd47a37b
+056c162c-73b6-4cff-a372-5977c14d79e7
+058a1d60-ce40-4fcd-8de9-e74b4eac1bf4
+058d24aa-aeb0-4ed7-b90f-b4bc6bf637c4
+0615d8e3-d38d-4602-9869-f26601771647
+063d25b6-e8d6-47a0-a5cb-4a85a93d80f3
+06828737-e134-4c27-b9c7-64b238f2144f
+06bbed15-89fb-4ee3-92db-ed099cd66c1b
+06c1db91-42ac-4f5e-8049-024995a2b20c
+06d082af-9435-44f1-a754-56df2e1e0218
+06db6e63-6dc5-4f4b-ab66-edcb7bdeabfa
+06e6568d-cc25-4409-bd55-c999561c2125
+06f0de8b-c447-41e4-aff5-73b36ce2d763
+0765b13d-8d51-4001-84f8-2f466282ec0a
+078db134-2b98-4e82-9964-dbfce9213558
+0790aec1-62a6-422a-918f-a199ba30e616
+07a2bdc8-eb67-4a33-af47-4da629d9967b
+07c03faf-a528-46de-9c53-a54a4fffbb8e
+07c1104d-4b8b-45db-97cb-f309a4257598
+07f9e2db-c32d-43c9-9a94-c1b14986f550
+085eefd2-0fb0-4635-91ae-2e787f24e772
+08875a08-f046-4cac-8b95-d5f124f3949b
+0895db75-9173-4002-b8aa-25e28ca8f082
+08b61437-2781-43f0-b984-8c65599cadb7
+08e63679-4812-4e2f-895d-3f4cfeac0f2b
+08f9526e-294a-41fa-bb4a-b6456df30ffd
+0900817a-a85e-4ae4-a48e-6e6c7d9db3f6
+091350dc-c50f-452d-8994-0c17ab592770
+091689c5-800c-487e-b944-77cb1b52f6ab
+09614ac1-d05d-4b7f-ae85-21c91333b4e9
+096b7528-e16e-4ec1-8547-af6acf881dc4
+09a3011b-8c20-427e-9804-aef8a0e4575f
+09b61041-21e3-4cdf-826a-f50f6073baa0
+09cf53f3-0617-422b-a3b8-37cc604d50d9
+09d32625-87ed-4109-a9f8-d12b11b11f20
+09e36bfb-b527-472a-9947-8810fd0aa78c
+09e53921-215b-418f-b866-bccc4edfe6b9
+09f5c5cd-ec62-4299-8351-b2bf76c7bd4a
+0a198682-6836-4f10-b7c0-46ca620d5103
+0a91da68-234c-46aa-8554-14e386f901fc
+0aa3aa8c-0b2c-4cbc-b53e-e7bff33506e6
+0aae8b44-9689-44f8-bdc4-75cd7d8f863f
+0ab56e1a-8e60-44ae-8130-36970b1c5d4a
+0ad567cd-f733-47a2-b13f-e753c1767cc2
+0ae4451e-89a4-48b4-9de8-83fe5678a79f
+0b03528b-4cb5-4e66-b871-5c200e529d1f
+0b9e6cf7-9d7e-4b98-b061-64d3ad6e075b
+0ba845e9-699f-47a3-9870-395299d2f359
+0bd30618-b404-4c18-9066-840ac2ce4812
+0c39b4e2-f78b-4f31-b6dd-c58505a2aefa
+0c46ef6d-30f6-479d-bf1e-2314e379e843
+0c4a25b9-7ef6-4607-a47f-256e6f30d86c
+0c52a734-8630-4309-a752-f2b1d130d657
+0c588cf9-84df-40c8-896d-c895cfa736d9
+0c5fdb86-88ba-4f89-b042-e7a6d5fff03e
+0cf9aad5-bb4a-461d-b2bd-699ad978d2c7
+0d419616-8a18-45b0-8f9e-ad30e3ba2887
+0d467189-117a-4b98-8515-67f57b3a8f31
+0d91b525-db8c-4bbd-b7a3-4bab45be8c8c
+0e035f76-86cb-45c4-89a8-03325d171df9
+0e0ed498-067c-4818-9bec-b76e4953cb0e
+0e1009a2-f522-4eeb-9356-b8683b4d49d3
+0e335e20-321b-4e41-8763-00c8c6e4930a
+0e909803-a39e-476a-8265-1f56d77d0b53
+0e9c1c91-f818-48d9-a8ad-6956b184b7b6
+0ea7e399-8766-47b8-952b-10f752e50872
+0eaba7b6-5853-4ef8-81c8-9437d75fe837
+0ecbd07a-5230-439e-a690-72feb8f4c2de
+0eebec4e-5a48-4fde-9ee4-af6a469a2c11
+0f04df6b-20be-4cc4-b7d0-65dceb1b56e4
+0f17b161-2218-4404-a681-95c02dea9485
+0f228d7b-5608-4f89-8967-ecfa4e05714c
+0f3f8a3a-bda0-46fc-8945-da18676d8496
+0f7eafc1-2062-4674-95be-286a33f4cf15
+0fbf77ea-3dfe-45ff-b37b-282e4862bc2f
+101c7808-ed59-45cb-90bc-8613ecfc6e31
+1021ddb3-b705-4387-9b5c-6375809b3e0f
+102c775a-8a33-4e39-ba77-41f54b1e6786
+104c0682-c09f-49ec-b71f-2f0731b74e93
+104f5c65-b19b-4283-a0eb-82bbe46a644e

kylin git commit: KYLIN-2195 prepare name mapping

2016-11-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master bce9ab822 -> 79cb59cfd


KYLIN-2195 prepare name mapping


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/79cb59cf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/79cb59cf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/79cb59cf

Branch: refs/heads/master
Commit: 79cb59cfda2caa9b4a623d2e425c6b3d99ea50db
Parents: bce9ab8
Author: Li Yang 
Authored: Tue Nov 22 18:36:20 2016 +0800
Committer: Li Yang 
Committed: Tue Nov 22 18:36:32 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   8 -
 .../kylin-backward-compatibility.properties | 187 ++-
 2 files changed, 186 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/79cb59cf/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3252245..a84ed62 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -785,14 +785,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.enable.scheduler", "0"));
 }
 
-public String getZookeeperAddress() {
-return this.getOptional("kylin.zookeeper.address");
-}
-
-public String getRestAddress() {
-return this.getOptional("kylin.rest.address", "localhost:7070");
-}
-
 private Map convertKeyToInteger(Map map) {
 Map result = Maps.newLinkedHashMap();
 for (Entry entry : map.entrySet()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/79cb59cf/examples/test_case_data/localmeta/kylin-backward-compatibility.properties
--
diff --git 
a/examples/test_case_data/localmeta/kylin-backward-compatibility.properties 
b/examples/test_case_data/localmeta/kylin-backward-compatibility.properties
index 67aef7a..30395df 100644
--- a/examples/test_case_data/localmeta/kylin-backward-compatibility.properties
+++ b/examples/test_case_data/localmeta/kylin-backward-compatibility.properties
@@ -1 +1,186 @@
-kylin.test.bcc.old.key=kylin.test.bcc.new.key
\ No newline at end of file
+
+### TEST ###
+
+kylin.test.bcc.old.key=kylin.test.bcc.new.key
+
+
+### ENV ###
+
+deploy.env=kylin.env
+kylin.hdfs.working.dir=kylin.env.hdfs-working-dir
+
+
+### METADATA ###
+
+kylin.metadata.url=kylin.metadata.url
+kylin.realization.providers=kylin.metadata.realization-providers
+kylin.cube.dimension.customEncodingFactories=kylin.metadata.custom-dimension-encodings
+kylin.cube.measure.customMeasureType.=kylin.metadata.custom-measure-type.
+
+
+### Dictionary ###
+
+kylin.table.snapshot.max_mb=kylin.snapshot.max-mb
+kylin.snapshot.cache.max.entry=kylin.snapshot.max-cache-entry
+kylin.dictionary.forest.trie.size.max_mb=kylin.dictionary.forest-trie-max-mb
+kylin.dict.cache.max.entry=kylin.dictionary.max-cache-entry
+kylin.dict.growing.enabled=kylin.dictionary.growing-enabled
+kylin.dict.append.entry.size=kylin.dictionary.append-entry-size
+kylin.dict.append.cache.size=kylin.dictionary.append-cache-size
+
+
+### CUBE ###
+
+kylin.job.cuboid.size.ratio=kylin.cube.size-estimate-ratio
+kylin.job.cuboid.size.memhungry.ratio=kylin.cube.size-estimate-memhungry-ratio
+kylin.cube.algorithm=kylin.cube.algorithm
+kylin.cube.algorithm.auto.threshold=kylin.cube.algorithm.layer-or-inmem-threshold
+kylin.cube.algorithm.auto.mapper.limit=kylin.cube.algorithm.inmem-split-limit
+kylin.cube.aggrgroup.max.size=kylin.cube.aggrgroup.max-size
+kylin.cube.aggrgroup.max.combination=kylin.cube.aggrgroup.max-combination
+kylin.cube.aggrgroup.isMandatoryOnlyValid=kylin.cube.aggrgroup.is-mandatory-only-valid
+kylin.cube.building.segment.max=kylin.cube.max-building-segments
+
+
+### JOB ###
+
+kylin.job.log.dir=kylin.job.log-dir
+kylin.job.remote.cli.working.dir=kylin.job.remote-cli-working-dir
+kylin.job.allow.empty.segment=kylin.job.allow-empty-segment
+kylin.job.concurrent.max.limit=kylin.job.max-concurrent-jobs
+kylin.job.cubing.inmem.sampling.percent=kylin.job.sampling-percentage
+kylin.job.cubing.inmem.sampling.hll.precision=kylin.job.sampling-hll-precision
+kylin.job.dependency.filterlist=kylin.job.dependency-filter-list
+kylin.job.retry=kylin.job.retry
+kylin.job.controller.lock=kylin.job.lock
+kylin.scheduler.=kylin.job.scheduler.provider.
+kylin.enable.scheduler=kylin.job.scheduler.default
+

kylin git commit: int deorecated show better

2016-11-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master a058049c8 -> bce9ab822


int deorecated show better

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bce9ab82
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bce9ab82
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bce9ab82

Branch: refs/heads/master
Commit: bce9ab822492772c7c2b42421941421dfb27f9f7
Parents: a058049
Author: luguosheng <550175...@qq.com>
Authored: Tue Nov 22 17:12:35 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 17:14:04 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bce9ab82/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 10f8185..888662d 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -49,7 +49,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   var name = value;
   var typeVersion=+encodings[i];
   if(value=="int"){
-name = "int deprecated";
+name = "int (deprecated)";
   }
   if(/\d+/.test(""+typeVersion)&>1){
   for(var s=1;s<=typeVersion;s++){

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-22 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1b83470
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1b83470
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1b83470

Branch: refs/heads/yang21-cdh5.7
Commit: e1b83470be3e8f315ad6c5001b97e62d5682dd34
Parents: 150bab6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 17:13:01 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1b83470/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-22 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1b83470/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1b83470/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[1/4] kylin git commit: revert KYLIN-2199 because KYLIN-2191 is not a major fix [Forced Update!]

2016-11-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 229cffdbd -> ac2d7c41d (forced update)


revert KYLIN-2199 because KYLIN-2191 is not a major fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/150bab61
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/150bab61
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/150bab61

Branch: refs/heads/yang21-cdh5.7
Commit: 150bab61a77c0900bf8d43f569b710d8b9177f0d
Parents: d4715b5
Author: Hongbin Ma 
Authored: Tue Nov 22 15:32:31 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 15:51:08 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |   4 +-
 .../dimension/DimensionEncodingFactory.java |   4 -
 .../apache/kylin/dimension/IntegerDimEnc.java   |   3 +-
 .../apache/kylin/dimension/IntegerDimEncV2.java | 228 ---
 .../kylin/dimension/IntegerDimEncTest.java  |  16 +-
 5 files changed, 11 insertions(+), 244 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index d572e56..87d37be 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.IntegerDimEncV2;
+import org.apache.kylin.dimension.IntegerDimEnc;
 import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -99,7 +99,7 @@ public class DimEncodingPreserveOrderTest {
 @Test
 public void testVLongDimEncPreserveOrder() {
 for (int i = 1; i <= successValue.size(); i++) {
-IntegerDimEncV2 enc = new IntegerDimEncV2(i);
+IntegerDimEnc enc = new IntegerDimEnc(i);
 List encodedValues = Lists.newArrayList();
 for (long value : successValue.get(i - 1)) {
 encodedValues.add(encode(enc, value));

http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index b617a19..2b6f894 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -117,10 +117,6 @@ public abstract class DimensionEncodingFactory {
 map.put(Pair.newPair(IntegerDimEnc.ENCODING_NAME, 
value.getCurrentVersion()), value);
 }
 {
-IntegerDimEncV2.Factory value = new IntegerDimEncV2.Factory();
-map.put(Pair.newPair(IntegerDimEncV2.ENCODING_NAME, 
value.getCurrentVersion()), value);
-}
-{
 FixedLenHexDimEnc.Factory value = new 
FixedLenHexDimEnc.Factory();
 map.put(Pair.newPair(FixedLenHexDimEnc.ENCODING_NAME, 
value.getCurrentVersion()), value);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 983af9a..bcd6283 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
  * -2^(8*N-1) is not supported because the slot is reserved for null values.
  * -2^(8*N-1) will be encoded with warn, and its output will be null
  */
-@Deprecated//due to a fatal bug (KYLIN-2191)
 public class IntegerDimEnc extends DimensionEncoding {
 private static final long serialVersionUID = 1L;
 
@@ -132,7 +131,7 @@ public class IntegerDimEnc extends DimensionEncoding {
 
 //only take useful bytes

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-22 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1b83470/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1b83470/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-22 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1b83470
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1b83470
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1b83470

Branch: refs/heads/yang21-hbase1.x
Commit: e1b83470be3e8f315ad6c5001b97e62d5682dd34
Parents: 150bab6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 17:13:01 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1b83470/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[1/3] kylin git commit: revert KYLIN-2199 because KYLIN-2191 is not a major fix [Forced Update!]

2016-11-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x edb9d54e9 -> e1b83470b (forced update)


revert KYLIN-2199 because KYLIN-2191 is not a major fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/150bab61
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/150bab61
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/150bab61

Branch: refs/heads/yang21-hbase1.x
Commit: 150bab61a77c0900bf8d43f569b710d8b9177f0d
Parents: d4715b5
Author: Hongbin Ma 
Authored: Tue Nov 22 15:32:31 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 15:51:08 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |   4 +-
 .../dimension/DimensionEncodingFactory.java |   4 -
 .../apache/kylin/dimension/IntegerDimEnc.java   |   3 +-
 .../apache/kylin/dimension/IntegerDimEncV2.java | 228 ---
 .../kylin/dimension/IntegerDimEncTest.java  |  16 +-
 5 files changed, 11 insertions(+), 244 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index d572e56..87d37be 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.IntegerDimEncV2;
+import org.apache.kylin.dimension.IntegerDimEnc;
 import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -99,7 +99,7 @@ public class DimEncodingPreserveOrderTest {
 @Test
 public void testVLongDimEncPreserveOrder() {
 for (int i = 1; i <= successValue.size(); i++) {
-IntegerDimEncV2 enc = new IntegerDimEncV2(i);
+IntegerDimEnc enc = new IntegerDimEnc(i);
 List encodedValues = Lists.newArrayList();
 for (long value : successValue.get(i - 1)) {
 encodedValues.add(encode(enc, value));

http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index b617a19..2b6f894 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -117,10 +117,6 @@ public abstract class DimensionEncodingFactory {
 map.put(Pair.newPair(IntegerDimEnc.ENCODING_NAME, 
value.getCurrentVersion()), value);
 }
 {
-IntegerDimEncV2.Factory value = new IntegerDimEncV2.Factory();
-map.put(Pair.newPair(IntegerDimEncV2.ENCODING_NAME, 
value.getCurrentVersion()), value);
-}
-{
 FixedLenHexDimEnc.Factory value = new 
FixedLenHexDimEnc.Factory();
 map.put(Pair.newPair(FixedLenHexDimEnc.ENCODING_NAME, 
value.getCurrentVersion()), value);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/150bab61/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 983af9a..bcd6283 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
  * -2^(8*N-1) is not supported because the slot is reserved for null values.
  * -2^(8*N-1) will be encoded with warn, and its output will be null
  */
-@Deprecated//due to a fatal bug (KYLIN-2191)
 public class IntegerDimEnc extends DimensionEncoding {
 private static final long serialVersionUID = 1L;
 
@@ -132,7 +131,7 @@ public class IntegerDimEnc extends DimensionEncoding {
 
 //only take useful bytes

kylin git commit: encoding version show better

2016-11-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 1c730b91b -> a058049c8


encoding version show better

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a058049c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a058049c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a058049c

Branch: refs/heads/master
Commit: a058049c8824db19c0a7bb724fc7ed3b920ff2d6
Parents: 1c730b9
Author: luguosheng <550175...@qq.com>
Authored: Tue Nov 22 16:30:22 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 16:50:34 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a058049c/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 9e2a3c9..10f8185 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -49,12 +49,12 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   var name = value;
   var typeVersion=+encodings[i];
   if(value=="int"){
-name = "int(deprecated)";
+name = "int deprecated";
   }
   if(/\d+/.test(""+typeVersion)&>1){
   for(var s=1;s<=typeVersion;s++){
 $scope.store.supportedEncoding.push({
-  "name":name+" 
(v"+s+")"+(s==typeVersion&>1?"(suggest)":""),
+  "name":name+" 
(v"+s+","+(s==typeVersion&>1?"suggest)":")"),
   "value":value+"[v"+s+"]",
   "version":typeVersion,
   "baseValue":value,

kylin git commit: revert KYLIN-2199 because KYLIN-2191 is not a major fix

2016-11-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 8132cab31 -> 1c730b91b


revert KYLIN-2199 because KYLIN-2191 is not a major fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c730b91
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c730b91
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c730b91

Branch: refs/heads/master
Commit: 1c730b91b3c0847873dbbf74106647b663be1942
Parents: 8132cab
Author: Hongbin Ma 
Authored: Tue Nov 22 15:32:31 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 15:32:31 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |   4 +-
 .../dimension/DimensionEncodingFactory.java |   4 -
 .../apache/kylin/dimension/IntegerDimEnc.java   |   3 +-
 .../apache/kylin/dimension/IntegerDimEncV2.java | 218 ---
 .../kylin/dimension/IntegerDimEncTest.java  |  16 +-
 5 files changed, 11 insertions(+), 234 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c730b91/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index 7d6af24..1866079 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -26,7 +26,7 @@ import java.util.List;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.IntegerDimEncV2;
+import org.apache.kylin.dimension.IntegerDimEnc;
 import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -98,7 +98,7 @@ public class DimEncodingPreserveOrderTest {
 @Test
 public void testVLongDimEncPreserveOrder() {
 for (int i = 1; i <= successValue.size(); i++) {
-IntegerDimEncV2 enc = new IntegerDimEncV2(i);
+IntegerDimEnc enc = new IntegerDimEnc(i);
 List encodedValues = Lists.newArrayList();
 for (long value : successValue.get(i - 1)) {
 encodedValues.add(encode(enc, value));

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c730b91/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 843f4cd..4954ead 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -117,10 +117,6 @@ public abstract class DimensionEncodingFactory {
 map.put(Pair.newPair(IntegerDimEnc.ENCODING_NAME, 
value.getCurrentVersion()), value);
 }
 {
-IntegerDimEncV2.Factory value = new IntegerDimEncV2.Factory();
-map.put(Pair.newPair(IntegerDimEncV2.ENCODING_NAME, 
value.getCurrentVersion()), value);
-}
-{
 FixedLenHexDimEnc.Factory value = new 
FixedLenHexDimEnc.Factory();
 map.put(Pair.newPair(FixedLenHexDimEnc.ENCODING_NAME, 
value.getCurrentVersion()), value);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c730b91/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 44d0e73..090dc83 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -36,7 +36,6 @@ import org.slf4j.LoggerFactory;
  * -2^(8*N-1) is not supported because the slot is reserved for null values.
  * -2^(8*N-1) will be encoded with warn, and its output will be null
  */
-@Deprecated//due to a fatal bug (KYLIN-2191) and the limitation of not able to 
represent -2^(8N-1)
 public class IntegerDimEnc extends DimensionEncoding {
 private static final long serialVersionUID = 1L;
 
@@ -122,7 +121,7 @@ public class IntegerDimEnc extends DimensionEncoding {
 
 //only take useful bytes

kylin git commit: KYLIN-2221 rethink on KYLIN-1684

2016-11-22 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 783d39917 -> 33bb7c293


KYLIN-2221 rethink on KYLIN-1684


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/33bb7c29
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/33bb7c29
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/33bb7c29

Branch: refs/heads/v1.6.0-rc2
Commit: 33bb7c293982a052a8c436820c708a038878e30f
Parents: 783d399
Author: Hongbin Ma 
Authored: Tue Nov 22 13:42:46 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 22 15:59:17 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 
 .../gtrecord/GTCubeStorageQueryBase.java| 26 
 2 files changed, 10 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/33bb7c29/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 011155f..c2595fb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -477,6 +477,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Float.parseFloat(getOptional("kylin.hbase.hfile.size.gb", 
"2.0"));
 }
 
+//check KYLIN-1684, in most cases keep the default value
+public boolean isSkippingEmptySegments() {
+return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
+}
+
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.pushdown.limit.max", 
"1"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/33bb7c29/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 76a8c7b..5e8af88 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -122,24 +122,12 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 List scanners = Lists.newArrayList();
 for (CubeSegment cubeSeg : 
cubeInstance.getSegments(SegmentStatusEnum.READY)) {
 CubeSegmentScanner scanner;
-if (cubeSeg.getInputRecords() == 0) {
-if (!skipZeroInputSegment(cubeSeg)) {
-logger.warn("cube segment {} input record is 0, " + "it 
may caused by kylin failed to get the job counter " + "as the hadoop history 
server wasn't running", cubeSeg);
-} else {
-logger.warn("cube segment {} input record is 0, skip it ", 
cubeSeg);
-continue;
-}
-}
-try {
-scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context, getGTStorage());
-} catch (IllegalArgumentException ex) {
-// ref KYLIN-1967, real empty segment can trigger dictionary 
exception -- IllegalArgumentException: Value not exists!
-if (cubeSeg.getInputRecords() == 0) {
-logger.warn("cube segment {} input record is 0, skip it 
still", cubeSeg);
-continue;
-}
-throw ex;
+if (cubeDesc.getConfig().isSkippingEmptySegments() && 
cubeSeg.getInputRecords() == 0) {
+logger.info("Skip cube segment {} because its input record is 
0", cubeSeg);
+continue;
 }
+
+scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context, getGTStorage());
 scanners.add(scanner);
 }
 
@@ -149,10 +137,6 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, 
metrics, returnTupleInfo, context);
 }
 
-protected boolean skipZeroInputSegment(CubeSegment cubeSegment) {
-return false;
-}
-
 protected abstract String getGTStorage();
 
 private void buildDimensionsAndMetrics(SQLDigest sqlDigest, 
Collection dimensions, Collection metrics) {

49 matches

Mail list logo