date:20161014

[06/10] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-10-14 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/c4e31c1b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 9b487a7..4a4f2a3 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -25,7 +25,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -78,7 +79,8 @@ public class CubeHTableUtil {
 tableDesc.setValue(IRealizationConstants.HTableSegmentTag, 
cubeSegment.toString());
 
 Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin admin = conn.getAdmin();
 
 try {
 if (User.isHBaseSecurityEnabled(conf)) {
@@ -91,7 +93,7 @@ public class CubeHTableUtil {
 tableDesc.addFamily(cf);
 }
 
-if (admin.tableExists(tableName)) {
+if (admin.tableExists(TableName.valueOf(tableName))) {
 // admin.disableTable(tableName);
 // admin.deleteTable(tableName);
 throw new RuntimeException("HBase table " + tableName + " 
exists!");
@@ -100,7 +102,7 @@ public class CubeHTableUtil {
 DeployCoprocessorCLI.deployCoprocessor(tableDesc);
 
 admin.createTable(tableDesc, splitKeys);
-Preconditions.checkArgument(admin.isTableAvailable(tableName), 
"table " + tableName + " created, but is not available due to some reasons");
+
Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)),
 "table " + tableName + " created, but is not available due to some reasons");
 logger.info("create hbase table " + tableName + " done.");
 } finally {
 admin.close();
@@ -109,8 +111,7 @@ public class CubeHTableUtil {
 }
 
 public static void deleteHTable(TableName tableName) throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = new HBaseAdmin(conf);
+Admin admin = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
 try {
 if (admin.tableExists(tableName)) {
 logger.info("disabling hbase table " + tableName);
@@ -125,8 +126,7 @@ public class CubeHTableUtil {
 
 /** create a HTable that has the same performance settings as normal cube 
table, for benchmark purpose */
 public static void createBenchmarkHTable(TableName tableName, String 
cfName) throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = new HBaseAdmin(conf);
+Admin admin = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
 try {
 if (admin.tableExists(tableName)) {
 logger.info("disabling hbase table " + tableName);

http://git-wip-us.apache.org/repos/asf/kylin/blob/c4e31c1b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 7aecd7e..9dc9715 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -28,9 +28,10 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.job.exception.ExecuteException;
 import

[05/10] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-10-14 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/c4e31c1b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 1d85922..04e2e8b 100644
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.FilterList.Operator;
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
@@ -136,7 +137,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
 
 Put p = new Put(rk);
 p.setDurability(Durability.SKIP_WAL);
-p.add(cf.getBytes(), cq, Bytes.toBytes(c));
+p.addColumn(cf.getBytes(), cq, Bytes.toBytes(c));
 ht.put(p);
 }
 }
@@ -224,7 +225,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
 scan.addFamily(cf.getBytes());
 scan.setFilter(filter);
 List regions = 
TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
-HRegion first = regions.get(0);
+Region first = regions.get(0);
 first.getScanner(scan);
 RegionScanner scanner = first.getScanner(scan);
 List results = new ArrayList();

[09/10] kylin git commit: BeelineHiveClient.java

2016-10-14 Thread mahongbin

BeelineHiveClient.java


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/13b7b542
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/13b7b542
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/13b7b542

Branch: refs/heads/1.5.4.1-beeline
Commit: 13b7b542d8fdf45de6aa7bcbeaf9a089bbc9
Parents: 3bd0c10
Author: Hongbin Ma 
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 13 15:31:11 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  16 +-
 .../kylin/rest/controller/TableController.java  |   9 +-
 source-hive/pom.xml |   5 +
 .../kylin/source/hive/BeelineHiveClient.java| 214 +++
 .../source/hive/BeelineOptionsProcessor.java|  47 
 .../apache/kylin/source/hive/CLIHiveClient.java | 169 +++
 .../apache/kylin/source/hive/HiveClient.java| 170 ---
 .../kylin/source/hive/HiveClientFactory.java|  33 +++
 .../source/hive/HiveSourceTableLoader.java  |  57 ++---
 .../org/apache/kylin/source/hive/HiveTable.java |  16 +-
 .../apache/kylin/source/hive/HiveTableMeta.java |  71 ++
 .../kylin/source/hive/HiveTableMetaBuilder.java | 102 +
 .../apache/kylin/source/hive/HqlExecutable.java | 107 --
 .../apache/kylin/source/hive/IHiveClient.java   |  36 
 .../hive/BeelineOptionsProcessorTest.java   |  38 
 15 files changed, 762 insertions(+), 328 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/13b7b542/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 8c64f91..7a8cbe4 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -18,7 +18,12 @@
 
 package org.apache.kylin.job;
 
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.List;
 
@@ -38,8 +43,9 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -48,9 +54,6 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 
-import kafka.message.Message;
-import kafka.message.MessageAndOffset;
-
 public class DeployUtil {
 private static final Logger logger = 
LoggerFactory.getLogger(DeployUtil.class);
 
@@ -179,7 +182,6 @@ public class DeployUtil {
 in.close();
 }
 
-
 private static void deployHiveTables() throws Exception {
 
 MetadataManager metaMgr = MetadataManager.getInstance(config());
@@ -205,7 +207,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/13b7b542/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index eefeba8..b7dd728 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.ModelService;
 import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import

[02/10] kylin git commit: Revert "KYLIN-2012 more robust approach to hive schema changes"

2016-10-14 Thread mahongbin

Revert "KYLIN-2012 more robust approach to hive schema changes"

This reverts commit 17569f6c32a373f599ef7689f9506b5af5ed68bd.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d33ba6aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d33ba6aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d33ba6aa

Branch: refs/heads/1.5.4.1-beeline
Commit: d33ba6aaaddf526b78225eab2a9280a39dd2058d
Parents: 18a1454
Author: shaofengshi 
Authored: Fri Sep 23 18:11:29 2016 +0800
Committer: shaofengshi 
Committed: Fri Sep 23 18:11:29 2016 +0800

--
 .../org/apache/kylin/cube/CubeDescManager.java  |  62 +++---
 .../org/apache/kylin/cube/CubeInstance.java |  11 +-
 .../java/org/apache/kylin/cube/CubeManager.java |  47 ++--
 .../org/apache/kylin/cube/model/CubeDesc.java   |  47 ++--
 .../model/validation/CubeMetadataValidator.java |  32 ++-
 .../realization/RealizationStatusEnum.java  |   2 +-
 .../kylin/rest/controller/CubeController.java   |  44 ++--
 .../apache/kylin/rest/service/CacheService.java |  11 +-
 .../apache/kylin/rest/service/CubeService.java  |  15 ++
 .../apache/kylin/rest/service/JobService.java   |   6 -
 .../kylin/rest/service/CubeServiceTest.java |   1 +
 .../source/hive/HiveSourceTableLoader.java  |  33 +--
 .../apache/kylin/source/hive/SchemaChecker.java | 216 ---
 webapp/app/css/AdminLTE.css |   4 +-
 webapp/app/partials/cubes/cubes.html|  22 +-
 15 files changed, 200 insertions(+), 353 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d33ba6aa/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 1b1cf70..33a6830 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.ResourceStore;
@@ -35,7 +36,6 @@ import 
org.apache.kylin.cube.model.validation.CubeMetadataValidator;
 import org.apache.kylin.cube.model.validation.ValidateContext;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -110,34 +110,30 @@ public class CubeDescManager {
  * @throws IOException
  */
 public CubeDesc reloadCubeDescLocal(String name) throws IOException {
-// Broken CubeDesc is not allowed to be saved and broadcast.
-CubeDesc ndesc = loadCubeDesc(CubeDesc.concatResourcePath(name), 
false);
 
-cubeDescMap.putLocal(ndesc.getName(), ndesc);
-Cuboid.reloadCache(name);
+// Save Source
+String path = CubeDesc.concatResourcePath(name);
 
-// if related cube is in DESCBROKEN state before, change it back to 
DISABLED
-CubeManager cubeManager = CubeManager.getInstance(config);
-for (CubeInstance cube : cubeManager.getCubesByDesc(name)) {
-if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
-cubeManager.reloadCubeLocal(cube.getName());
-}
-}
+// Reload the CubeDesc
+CubeDesc ndesc = loadCubeDesc(path);
 
+// Here replace the old one
+cubeDescMap.putLocal(ndesc.getName(), ndesc);
+Cuboid.reloadCache(name);
 return ndesc;
 }
 
-private CubeDesc loadCubeDesc(String path, boolean allowBroken) throws 
IOException {
+private CubeDesc loadCubeDesc(String path) throws IOException {
 ResourceStore store = getStore();
 CubeDesc ndesc = store.getResource(path, CubeDesc.class, 
CUBE_DESC_SERIALIZER);
 
-try {
-ndesc.init(config, getMetadataManager().getAllTablesMap());
-} catch (Exception e) {
-ndesc.addError(e.getMessage());
+if (StringUtils.isBlank(ndesc.getName())) {
+throw new IllegalStateException("CubeDesc name must not be blank");
 }
 
-if (!allowBroken && !ndesc.getError().isEmpty()) {
+ndesc.init(config, getMetadataManager().getAllTablesMap());
+
+if (ndesc.getError().isEmpty() == false) {
 throw new IllegalStateException("Cube desc at " + path + " has 
issues: "

[07/10] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-10-14 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c4e31c1b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c4e31c1b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c4e31c1b

Branch: refs/heads/1.5.4.1-beeline
Commit: c4e31c1b3a664f598352061ae8703812e9d9cef7
Parents: b08871e
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 14:46:35 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  |  19 +-
 .../test_case_data/sandbox/kylin_job_conf.xml   |  86 ++---
 examples/test_case_data/sandbox/mapred-site.xml |  23 +-
 .../kylin/provision/BuildCubeWithEngine.java|  53 +--
 pom.xml |  20 +-
 .../kylin/rest/security/AclHBaseStorage.java|   4 +-
 .../rest/security/MockAclHBaseStorage.java  |   8 +-
 .../apache/kylin/rest/security/MockHTable.java  |  95 +
 .../rest/security/RealAclHBaseStorage.java  |   9 +-
 .../apache/kylin/rest/service/AclService.java   |  25 +-
 .../apache/kylin/rest/service/CubeService.java  |  36 +-
 .../apache/kylin/rest/service/QueryService.java |  24 +-
 .../apache/kylin/rest/service/UserService.java  |  17 +-
 .../kylin/storage/hbase/HBaseConnection.java| 100 ++---
 .../kylin/storage/hbase/HBaseResourceStore.java |  31 +-
 .../storage/hbase/cube/SimpleHBaseStore.java|  20 +-
 .../hbase/cube/v1/CubeSegmentTupleIterator.java |  11 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |   4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java |  10 +-
 .../cube/v1/SerializedHBaseTupleIterator.java   |   4 +-
 .../observer/AggregateRegionObserver.java   |   4 +-
 .../observer/AggregationScanner.java|  14 +-
 .../observer/ObserverAggregationCache.java  |  10 +-
 .../coprocessor/observer/ObserverEnabler.java   |   4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  10 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |   9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |   4 +-
 .../storage/hbase/steps/CubeHTableUtil.java |  16 +-
 .../storage/hbase/steps/DeprecatedGCStep.java   |  23 +-
 .../storage/hbase/steps/HBaseCuboidWriter.java  |   8 +-
 .../hbase/steps/HBaseStreamingOutput.java   |   9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  |  23 +-
 .../storage/hbase/util/CleanHtableCLI.java  |  12 +-
 .../storage/hbase/util/CubeMigrationCLI.java| 372 +--
 .../hbase/util/CubeMigrationCheckCLI.java   |  19 +-
 .../hbase/util/DeployCoprocessorCLI.java|  22 +-
 .../hbase/util/ExtendCubeToHybridCLI.java   |   8 +-
 .../hbase/util/GridTableHBaseBenchmark.java |  34 +-
 .../kylin/storage/hbase/util/HBaseClean.java|  18 +-
 .../hbase/util/HBaseRegionSizeCalculator.java   |  35 +-
 .../kylin/storage/hbase/util/HBaseUsage.java|   9 +-
 .../storage/hbase/util/HbaseStreamingInput.java |  30 +-
 .../hbase/util/HtableAlterMetadataCLI.java  |   9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java |  17 +-
 .../kylin/storage/hbase/util/PingHBaseCLI.java  |  15 +-
 .../kylin/storage/hbase/util/RowCounterCLI.java |  11 +-
 .../storage/hbase/util/StorageCleanupJob.java   |  20 +-
 .../storage/hbase/util/UpdateHTableHostCLI.java |  17 +-
 .../observer/AggregateRegionObserverTest.java   |  31 +-
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |   5 +-
 50 files changed, 657 insertions(+), 760 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c4e31c1b/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten

[10/10] kylin git commit: remove fitler hive dependency logic in AbstractHadoopJob

2016-10-14 Thread mahongbin

remove fitler hive dependency logic in AbstractHadoopJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8f4ed727
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8f4ed727
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8f4ed727

Branch: refs/heads/1.5.4.1-beeline
Commit: 8f4ed7279173e1e28c51039b8ba19f4e493cbaf8
Parents: 13b7b54
Author: Hongbin Ma 
Authored: Thu Oct 13 21:35:49 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 13 21:35:49 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8f4ed727/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index af2ed9f..8e5e7da 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -199,7 +199,8 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
 logger.info("Hive Dependencies Before Filtered: " + 
kylinHiveDependency);
-String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+//String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+String filteredHive = kylinHiveDependency;
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
 if (kylinDependency.length() > 0)

[08/10] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-10-14 Thread mahongbin

KYLIN-1672 support kylin on cdh 5.7


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3bd0c10f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3bd0c10f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3bd0c10f

Branch: refs/heads/1.5.4.1-beeline
Commit: 3bd0c10fe363fd309377accd73e584cea1d0fdba
Parents: c4e31c1
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: shaofengshi 
Committed: Tue Sep 27 15:30:05 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 396 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  23 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|   8 +
 11 files changed, 431 insertions(+), 1129 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3bd0c10f/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/3bd0c10f/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3bd0c10f/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[04/10] kylin git commit: [maven-release-plugin] prepare release kylin-1.5.4.1

2016-10-14 Thread mahongbin

[maven-release-plugin] prepare release kylin-1.5.4.1


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b08871e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b08871e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b08871e7

Branch: refs/heads/1.5.4.1-beeline
Commit: b08871e7dd4595474832b5b6a2e156935b2d73c6
Parents: 2141b21
Author: shaofengshi 
Authored: Sat Sep 24 00:01:07 2016 +
Committer: shaofengshi 
Committed: Sat Sep 24 00:01:07 2016 +

--
 assembly/pom.xml |  5 ++---
 atopcalcite/pom.xml  |  5 ++---
 core-common/pom.xml  |  5 ++---
 core-cube/pom.xml|  5 ++---
 core-dictionary/pom.xml  |  5 ++---
 core-job/pom.xml |  5 ++---
 core-metadata/pom.xml|  5 ++---
 core-storage/pom.xml |  5 ++---
 engine-mr/pom.xml|  5 ++---
 engine-spark/pom.xml |  5 ++---
 engine-streaming/pom.xml |  5 ++---
 jdbc/pom.xml |  5 ++---
 kylin-it/pom.xml | 13 ++---
 pom.xml  | 11 +--
 query/pom.xml|  5 ++---
 server-base/pom.xml  |  5 ++---
 server/pom.xml   |  5 ++---
 source-hive/pom.xml  |  5 ++---
 source-kafka/pom.xml |  5 ++---
 storage-hbase/pom.xml|  8 +++-
 tomcat-ext/pom.xml   |  6 ++
 tool/pom.xml |  5 ++---
 22 files changed, 52 insertions(+), 76 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b08871e7/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 151ab0d..9bb8477 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -17,12 +17,11 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
 kylin
 org.apache.kylin
-1.5.4.1-SNAPSHOT
+1.5.4.1
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/b08871e7/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 744dbb0..50780f2 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 atopcalcite
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.4.1-SNAPSHOT
+1.5.4.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b08871e7/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 641f88d..32c9b67 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 kylin-core-common
@@ -28,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.5.4.1-SNAPSHOT
+1.5.4.1
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b08871e7/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 54f4786..d863b1d 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -17,8 +17,7 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0

[03/10] kylin git commit: add .gitconfig to rat exclude list

2016-10-14 Thread mahongbin

add .gitconfig to rat exclude list


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2141b214
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2141b214
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2141b214

Branch: refs/heads/1.5.4.1-beeline
Commit: 2141b214c1659d005f23b3587de6587d51eec1ff
Parents: d33ba6a
Author: shaofengshi 
Authored: Fri Sep 23 23:56:02 2016 +
Committer: shaofengshi 
Committed: Fri Sep 23 23:56:02 2016 +

--
 pom.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2141b214/pom.xml
--
diff --git a/pom.xml b/pom.xml
index fc5b511..d2bd442 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1131,6 +1131,7 @@
 **/LICENSE*
 
 .git/**
+.gitconfig
 
 .idea/**
 **/*.iml
@@ -1272,4 +1273,4 @@
 
 
 
-
\ No newline at end of file
+

[01/10] kylin git commit: update version

2016-10-14 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/1.5.4.1-beeline [created] 8f4ed7279


update version


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/18a14547
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/18a14547
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/18a14547

Branch: refs/heads/1.5.4.1-beeline
Commit: 18a14547cb923d6c051f48a774cb8b804ad2d510
Parents: 4c9b67f
Author: shaofengshi 
Authored: Fri Sep 23 16:57:26 2016 +0800
Committer: shaofengshi 
Committed: Fri Sep 23 16:57:26 2016 +0800

--
 assembly/pom.xml | 2 +-
 atopcalcite/pom.xml  | 2 +-
 core-common/pom.xml  | 2 +-
 core-cube/pom.xml| 2 +-
 core-dictionary/pom.xml  | 2 +-
 core-job/pom.xml | 2 +-
 core-metadata/pom.xml| 2 +-
 core-storage/pom.xml | 2 +-
 engine-mr/pom.xml| 2 +-
 engine-spark/pom.xml | 2 +-
 engine-streaming/pom.xml | 2 +-
 jdbc/pom.xml | 2 +-
 kylin-it/pom.xml | 2 +-
 pom.xml  | 2 +-
 query/pom.xml| 2 +-
 server-base/pom.xml  | 2 +-
 server/pom.xml   | 2 +-
 source-hive/pom.xml  | 2 +-
 source-kafka/pom.xml | 2 +-
 storage-hbase/pom.xml| 2 +-
 tomcat-ext/pom.xml   | 2 +-
 tool/pom.xml | 2 +-
 22 files changed, 22 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/18a14547/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index c7bde96..151ab0d 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
 
 kylin
 org.apache.kylin
-1.6.0-SNAPSHOT
+1.5.4.1-SNAPSHOT
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/18a14547/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 177b933..744dbb0 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.5.4.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/18a14547/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 780fd82..641f88d 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.5.4.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/18a14547/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 7c9a549..54f4786 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.5.4.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/18a14547/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 1bf97af..d4940b9 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.5.4.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/18a14547/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 601f730..a250b07 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.5.4.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/18a14547/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 142dd33..6948eba 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.5.4.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/18a14547/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 2d9d618..e801c9d 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-1.6.0-SNAPSHOT
+1.5.4.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/18a14547/engine-mr/pom.xml

[06/10] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[05/10] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[09/10] kylin git commit: BeelineHiveClient.java

[02/10] kylin git commit: Revert "KYLIN-2012 more robust approach to hive schema changes"

[07/10] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[10/10] kylin git commit: remove fitler hive dependency logic in AbstractHadoopJob

[08/10] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[04/10] kylin git commit: [maven-release-plugin] prepare release kylin-1.5.4.1

[03/10] kylin git commit: add .gitconfig to rat exclude list

[01/10] kylin git commit: update version

10 matches

Site Navigation

Mail list logo

Footer information