date:20170110

[2/6] kylin git commit: rollback ITEM_COUNT to INT type

2017-01-10 Thread billyliu

rollback ITEM_COUNT to INT type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a3ee3c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a3ee3c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a3ee3c1

Branch: refs/heads/yang22-hbase102
Commit: 3a3ee3c1cae56d4fed6948d64de23178c1a2cd1e
Parents: fb42404
Author: Yang Li 
Authored: Wed Jan 11 07:15:24 2017 +0800
Committer: Yang Li 
Committed: Wed Jan 11 07:15:24 2017 +0800

--
 .../localmeta/table/DEFAULT.TEST_KYLIN_FACT.json  | 2 +-
 .../test/java/org/apache/kylin/query/HackedDbUnitAssert.java  | 7 ++-
 2 files changed, 3 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3ee3c1/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 53dc96b..fdfb322 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -33,7 +33,7 @@
   }, {
 "id" : "8",
 "name" : "ITEM_COUNT",
-"datatype" : "bigint"
+"datatype" : "int"
   }, {
 "id" : "9",
 "name" : "SELLER_ID",

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3ee3c1/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index c295430..338f698 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -70,11 +70,8 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 //throw error;
 //}
 
-// if both tables are empty, it is not necessary to compare columns, as
-// such
-// comparison
-// can fail if column metadata is different (which could occurs when
-// comparing empty tables)
+// if both tables are empty, it is not necessary to compare columns, 
as such comparison
+// can fail if column metadata is different (which could occurs when 
comparing empty tables)
 if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
 logger.debug("Tables are empty, hence equals.");
 return;

[1/6] kylin git commit: alter query test case to fix CI [Forced Update!]

2017-01-10 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 69e3ea42d -> 9c093f78a (forced update)


alter query test case to fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb424042
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb424042
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb424042

Branch: refs/heads/yang22-hbase102
Commit: fb424042ef3e29efe26e90874e0ff425148b89b6
Parents: 2fec060
Author: Li Yang 
Authored: Tue Jan 10 15:27:14 2017 +0800
Committer: Li Yang 
Committed: Tue Jan 10 15:27:14 2017 +0800

--
 .../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json| 2 +-
 kylin-it/src/test/resources/query/sql_limit/query01.sql| 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fb424042/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index fdfb322..53dc96b 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -33,7 +33,7 @@
   }, {
 "id" : "8",
 "name" : "ITEM_COUNT",
-"datatype" : "int"
+"datatype" : "bigint"
   }, {
 "id" : "9",
 "name" : "SELLER_ID",

http://git-wip-us.apache.org/repos/asf/kylin/blob/fb424042/kylin-it/src/test/resources/query/sql_limit/query01.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_limit/query01.sql 
b/kylin-it/src/test/resources/query/sql_limit/query01.sql
index fca8175..296863f 100644
--- a/kylin-it/src/test/resources/query/sql_limit/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_limit/query01.sql
@@ -16,6 +16,6 @@
 -- limitations under the License.
 --
 
-select * from test_kylin_fact
+select cal_dt, lstg_format_name, seller_id from test_kylin_fact
   where lstg_format_name='FP-GTC' 
  limit 20

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-10 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/288a3dc3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/288a3dc3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[6/6] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2017-01-10 Thread billyliu

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a70112c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a70112c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a70112c

Branch: refs/heads/yang22-cdh5.7
Commit: 1a70112ce73f867db45ac5328a4695a570c01d73
Parents: 288a3dc
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Billy Liu 
Committed: Wed Jan 11 15:53:19 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1a70112c/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a70112c/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a70112c/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[5/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-10 Thread billyliu

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/288a3dc3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/288a3dc3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/288a3dc3

Branch: refs/heads/yang22-cdh5.7
Commit: 288a3dc3d38a46e457e002d2bb514f6a7a478a53
Parents: 2e08815
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Wed Jan 11 15:51:19 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/288a3dc3/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/6] kylin git commit: alter query test case to fix CI [Forced Update!]

2017-01-10 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 5ff6be9a1 -> 1a70112ce (forced update)


alter query test case to fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb424042
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb424042
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb424042

Branch: refs/heads/yang22-cdh5.7
Commit: fb424042ef3e29efe26e90874e0ff425148b89b6
Parents: 2fec060
Author: Li Yang 
Authored: Tue Jan 10 15:27:14 2017 +0800
Committer: Li Yang 
Committed: Tue Jan 10 15:27:14 2017 +0800

--
 .../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json| 2 +-
 kylin-it/src/test/resources/query/sql_limit/query01.sql| 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fb424042/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index fdfb322..53dc96b 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -33,7 +33,7 @@
   }, {
 "id" : "8",
 "name" : "ITEM_COUNT",
-"datatype" : "int"
+"datatype" : "bigint"
   }, {
 "id" : "9",
 "name" : "SELLER_ID",

http://git-wip-us.apache.org/repos/asf/kylin/blob/fb424042/kylin-it/src/test/resources/query/sql_limit/query01.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_limit/query01.sql 
b/kylin-it/src/test/resources/query/sql_limit/query01.sql
index fca8175..296863f 100644
--- a/kylin-it/src/test/resources/query/sql_limit/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_limit/query01.sql
@@ -16,6 +16,6 @@
 -- limitations under the License.
 --
 
-select * from test_kylin_fact
+select cal_dt, lstg_format_name, seller_id from test_kylin_fact
   where lstg_format_name='FP-GTC' 
  limit 20

[2/6] kylin git commit: rollback ITEM_COUNT to INT type

2017-01-10 Thread billyliu

rollback ITEM_COUNT to INT type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a3ee3c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a3ee3c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a3ee3c1

Branch: refs/heads/yang22-cdh5.7
Commit: 3a3ee3c1cae56d4fed6948d64de23178c1a2cd1e
Parents: fb42404
Author: Yang Li 
Authored: Wed Jan 11 07:15:24 2017 +0800
Committer: Yang Li 
Committed: Wed Jan 11 07:15:24 2017 +0800

--
 .../localmeta/table/DEFAULT.TEST_KYLIN_FACT.json  | 2 +-
 .../test/java/org/apache/kylin/query/HackedDbUnitAssert.java  | 7 ++-
 2 files changed, 3 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3ee3c1/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 53dc96b..fdfb322 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -33,7 +33,7 @@
   }, {
 "id" : "8",
 "name" : "ITEM_COUNT",
-"datatype" : "bigint"
+"datatype" : "int"
   }, {
 "id" : "9",
 "name" : "SELLER_ID",

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3ee3c1/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index c295430..338f698 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -70,11 +70,8 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 //throw error;
 //}
 
-// if both tables are empty, it is not necessary to compare columns, as
-// such
-// comparison
-// can fail if column metadata is different (which could occurs when
-// comparing empty tables)
+// if both tables are empty, it is not necessary to compare columns, 
as such comparison
+// can fail if column metadata is different (which could occurs when 
comparing empty tables)
 if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
 logger.debug("Tables are empty, hence equals.");
 return;

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-10 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/288a3dc3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/288a3dc3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[3/6] kylin git commit: KYLIN-2380 refactor DbUnit assertion

2017-01-10 Thread billyliu

KYLIN-2380 refactor DbUnit assertion


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e088159
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e088159
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e088159

Branch: refs/heads/yang22-cdh5.7
Commit: 2e0881596e251c7e517748c1468d459ec86210d3
Parents: 3a3ee3c
Author: Li Yang 
Authored: Wed Jan 11 12:10:36 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 12:10:36 2017 +0800

--
 .../apache/kylin/query/HackedDbUnitAssert.java  | 125 ---
 .../apache/kylin/query/ITMassInQueryTest.java   |   9 +-
 .../org/apache/kylin/query/KylinTestBase.java   |  60 +
 3 files changed, 148 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e088159/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index 338f698..3a21570 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -26,16 +26,34 @@ import org.dbunit.dataset.Columns;
 import org.dbunit.dataset.DataSetException;
 import org.dbunit.dataset.ITable;
 import org.dbunit.dataset.ITableMetaData;
+import org.dbunit.dataset.datatype.BigIntegerDataType;
 import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.IntegerDataType;
+import org.dbunit.dataset.datatype.UnknownDataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * dirty hack to support checking result of SQL with limit
+ * A copy of DbUnitAssert, in which a few hacks applied
+ * 
+ * - tolerate some column type difference, like INT vs. BIGINT
+ * - check expected table contains actual table (instead of equals), for sql 
with limit
  */
 public class HackedDbUnitAssert extends DbUnitAssert {
 private static final Logger logger = 
LoggerFactory.getLogger(HackedDbUnitAssert.class);
 
+private boolean hackCheckContains;
+private boolean hackIgnoreIntBigIntMismatch;
+
+public void hackCheckContains() {
+hackCheckContains = true;
+}
+
+public void hackIgnoreIntBigIntMismatch() {
+hackIgnoreIntBigIntMismatch = true;
+}
+
+// THIS METHOD IS MOSTLY COPIED FROM DbUnitAssert. CHANGES ARE LEAD BY 
hackXXX CONDITION CHECKS.
 public void assertEquals(ITable expectedTable, ITable actualTable, 
FailureHandler failureHandler) throws DatabaseUnitException {
 logger.trace("assertEquals(expectedTable, actualTable, failureHandler) 
- start");
 logger.debug("assertEquals: expectedTable={}", expectedTable);
@@ -57,22 +75,21 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 ITableMetaData actualMetaData = actualTable.getTableMetaData();
 String expectedTableName = expectedMetaData.getTableName();
 
-//// Verify row count
-//int expectedRowsCount = expectedTable.getRowCount();
-//int actualRowsCount = actualTable.getRowCount();
-//if (expectedRowsCount != actualRowsCount) {
-//String msg = "row count (table=" + expectedTableName + 
")";
-//Error error =
-//failureHandler.createFailure(msg, String
-//.valueOf(expectedRowsCount), String
-//.valueOf(actualRowsCount));
-//logger.error(error.toString());
-//throw error;
-//}
+// Verify row count
+int expectedRowsCount = expectedTable.getRowCount();
+int actualRowsCount = actualTable.getRowCount();
+if (!hackCheckContains) {
+if (expectedRowsCount != actualRowsCount) {
+String msg = "row count (table=" + expectedTableName + ")";
+Error error = failureHandler.createFailure(msg, 
String.valueOf(expectedRowsCount), String.valueOf(actualRowsCount));
+logger.error(error.toString());
+throw error;
+}
+}
 
 // if both tables are empty, it is not necessary to compare columns, 
as such comparison
 // can fail if column metadata is different (which could occurs when 
comparing empty tables)
-if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
+if (expectedRowsCount == 0 && actualRowsCount == 0) {
 logger.debug("Tables are empty, hence equals.");
 return;
 }
@@ -94,10 +111,86 @@ public

[5/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-10 Thread billyliu

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/288a3dc3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/288a3dc3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/288a3dc3

Branch: refs/heads/yang22-hbase1.x
Commit: 288a3dc3d38a46e457e002d2bb514f6a7a478a53
Parents: 2e08815
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Wed Jan 11 15:51:19 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/288a3dc3/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[3/5] kylin git commit: KYLIN-2380 refactor DbUnit assertion

2017-01-10 Thread billyliu

KYLIN-2380 refactor DbUnit assertion


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e088159
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e088159
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e088159

Branch: refs/heads/yang22-hbase1.x
Commit: 2e0881596e251c7e517748c1468d459ec86210d3
Parents: 3a3ee3c
Author: Li Yang 
Authored: Wed Jan 11 12:10:36 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 12:10:36 2017 +0800

--
 .../apache/kylin/query/HackedDbUnitAssert.java  | 125 ---
 .../apache/kylin/query/ITMassInQueryTest.java   |   9 +-
 .../org/apache/kylin/query/KylinTestBase.java   |  60 +
 3 files changed, 148 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e088159/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index 338f698..3a21570 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -26,16 +26,34 @@ import org.dbunit.dataset.Columns;
 import org.dbunit.dataset.DataSetException;
 import org.dbunit.dataset.ITable;
 import org.dbunit.dataset.ITableMetaData;
+import org.dbunit.dataset.datatype.BigIntegerDataType;
 import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.IntegerDataType;
+import org.dbunit.dataset.datatype.UnknownDataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * dirty hack to support checking result of SQL with limit
+ * A copy of DbUnitAssert, in which a few hacks applied
+ * 
+ * - tolerate some column type difference, like INT vs. BIGINT
+ * - check expected table contains actual table (instead of equals), for sql 
with limit
  */
 public class HackedDbUnitAssert extends DbUnitAssert {
 private static final Logger logger = 
LoggerFactory.getLogger(HackedDbUnitAssert.class);
 
+private boolean hackCheckContains;
+private boolean hackIgnoreIntBigIntMismatch;
+
+public void hackCheckContains() {
+hackCheckContains = true;
+}
+
+public void hackIgnoreIntBigIntMismatch() {
+hackIgnoreIntBigIntMismatch = true;
+}
+
+// THIS METHOD IS MOSTLY COPIED FROM DbUnitAssert. CHANGES ARE LEAD BY 
hackXXX CONDITION CHECKS.
 public void assertEquals(ITable expectedTable, ITable actualTable, 
FailureHandler failureHandler) throws DatabaseUnitException {
 logger.trace("assertEquals(expectedTable, actualTable, failureHandler) 
- start");
 logger.debug("assertEquals: expectedTable={}", expectedTable);
@@ -57,22 +75,21 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 ITableMetaData actualMetaData = actualTable.getTableMetaData();
 String expectedTableName = expectedMetaData.getTableName();
 
-//// Verify row count
-//int expectedRowsCount = expectedTable.getRowCount();
-//int actualRowsCount = actualTable.getRowCount();
-//if (expectedRowsCount != actualRowsCount) {
-//String msg = "row count (table=" + expectedTableName + 
")";
-//Error error =
-//failureHandler.createFailure(msg, String
-//.valueOf(expectedRowsCount), String
-//.valueOf(actualRowsCount));
-//logger.error(error.toString());
-//throw error;
-//}
+// Verify row count
+int expectedRowsCount = expectedTable.getRowCount();
+int actualRowsCount = actualTable.getRowCount();
+if (!hackCheckContains) {
+if (expectedRowsCount != actualRowsCount) {
+String msg = "row count (table=" + expectedTableName + ")";
+Error error = failureHandler.createFailure(msg, 
String.valueOf(expectedRowsCount), String.valueOf(actualRowsCount));
+logger.error(error.toString());
+throw error;
+}
+}
 
 // if both tables are empty, it is not necessary to compare columns, 
as such comparison
 // can fail if column metadata is different (which could occurs when 
comparing empty tables)
-if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
+if (expectedRowsCount == 0 && actualRowsCount == 0) {
 logger.debug("Tables are empty, hence equals.");
 return;
 }
@@ -94,10 +111,86 @@ public

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-10 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/288a3dc3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/288a3dc3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[2/5] kylin git commit: rollback ITEM_COUNT to INT type

2017-01-10 Thread billyliu

rollback ITEM_COUNT to INT type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a3ee3c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a3ee3c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a3ee3c1

Branch: refs/heads/yang22-hbase1.x
Commit: 3a3ee3c1cae56d4fed6948d64de23178c1a2cd1e
Parents: fb42404
Author: Yang Li 
Authored: Wed Jan 11 07:15:24 2017 +0800
Committer: Yang Li 
Committed: Wed Jan 11 07:15:24 2017 +0800

--
 .../localmeta/table/DEFAULT.TEST_KYLIN_FACT.json  | 2 +-
 .../test/java/org/apache/kylin/query/HackedDbUnitAssert.java  | 7 ++-
 2 files changed, 3 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3ee3c1/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 53dc96b..fdfb322 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -33,7 +33,7 @@
   }, {
 "id" : "8",
 "name" : "ITEM_COUNT",
-"datatype" : "bigint"
+"datatype" : "int"
   }, {
 "id" : "9",
 "name" : "SELLER_ID",

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3ee3c1/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index c295430..338f698 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -70,11 +70,8 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 //throw error;
 //}
 
-// if both tables are empty, it is not necessary to compare columns, as
-// such
-// comparison
-// can fail if column metadata is different (which could occurs when
-// comparing empty tables)
+// if both tables are empty, it is not necessary to compare columns, 
as such comparison
+// can fail if column metadata is different (which could occurs when 
comparing empty tables)
 if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
 logger.debug("Tables are empty, hence equals.");
 return;

[1/5] kylin git commit: alter query test case to fix CI [Forced Update!]

2017-01-10 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 081bec0b1 -> 288a3dc3d (forced update)


alter query test case to fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb424042
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb424042
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb424042

Branch: refs/heads/yang22-hbase1.x
Commit: fb424042ef3e29efe26e90874e0ff425148b89b6
Parents: 2fec060
Author: Li Yang 
Authored: Tue Jan 10 15:27:14 2017 +0800
Committer: Li Yang 
Committed: Tue Jan 10 15:27:14 2017 +0800

--
 .../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json| 2 +-
 kylin-it/src/test/resources/query/sql_limit/query01.sql| 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fb424042/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index fdfb322..53dc96b 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -33,7 +33,7 @@
   }, {
 "id" : "8",
 "name" : "ITEM_COUNT",
-"datatype" : "int"
+"datatype" : "bigint"
   }, {
 "id" : "9",
 "name" : "SELLER_ID",

http://git-wip-us.apache.org/repos/asf/kylin/blob/fb424042/kylin-it/src/test/resources/query/sql_limit/query01.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_limit/query01.sql 
b/kylin-it/src/test/resources/query/sql_limit/query01.sql
index fca8175..296863f 100644
--- a/kylin-it/src/test/resources/query/sql_limit/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_limit/query01.sql
@@ -16,6 +16,6 @@
 -- limitations under the License.
 --
 
-select * from test_kylin_fact
+select cal_dt, lstg_format_name, seller_id from test_kylin_fact
   where lstg_format_name='FP-GTC' 
  limit 20

kylin git commit: minor, change Dictionary to Dictionary in DictionaryManager and DictionaryInfo

2017-01-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 5f6450d6e -> be11dc629


minor,change Dictionary to Dictionary in DictionaryManager and 
DictionaryInfo

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be11dc62
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be11dc62
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be11dc62

Branch: refs/heads/master
Commit: be11dc629460ef4620300e1508c548037540f961
Parents: 5f6450d
Author: xiefan46 <958034...@qq.com>
Authored: Tue Jan 10 16:33:25 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 13:04:36 2017 +0800

--
 .../java/org/apache/kylin/dict/DictionaryInfo.java|  6 +++---
 .../apache/kylin/dict/DictionaryInfoSerializer.java   |  4 ++--
 .../java/org/apache/kylin/dict/DictionaryManager.java | 14 +++---
 .../kylin/engine/mr/steps/MergeCuboidMapperTest.java  |  6 +++---
 4 files changed, 15 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index 8526467..a85628d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -47,7 +47,7 @@ public class DictionaryInfo extends RootPersistentEntity {
 @JsonProperty("cardinality")
 private int cardinality;
 
-transient Dictionary dictionaryObject;
+transient Dictionary dictionaryObject;
 
 public DictionaryInfo() {
 }
@@ -144,11 +144,11 @@ public class DictionaryInfo extends RootPersistentEntity {
 this.dictionaryClass = dictionaryClass;
 }
 
-public Dictionary getDictionaryObject() {
+public Dictionary getDictionaryObject() {
 return dictionaryObject;
 }
 
-public void setDictionaryObject(Dictionary dictionaryObject) {
+public void setDictionaryObject(Dictionary dictionaryObject) {
 this.dictionaryObject = dictionaryObject;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
index 04eedbe..ee88ea7 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
@@ -61,9 +61,9 @@ public class DictionaryInfoSerializer implements 
Serializer {
 DictionaryInfo obj = JsonUtil.readValue(json, DictionaryInfo.class);
 
 if (infoOnly == false) {
-Dictionary dict;
+Dictionary dict;
 try {
-dict = (Dictionary) 
ClassUtil.forName(obj.getDictionaryClass(), Dictionary.class).newInstance();
+dict = (Dictionary) 
ClassUtil.forName(obj.getDictionaryClass(), Dictionary.class).newInstance();
 } catch (InstantiationException e) {
 throw new RuntimeException(e);
 } catch (IllegalAccessException e) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 54bc1c4..3ba24cf 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -108,7 +108,7 @@ public class DictionaryManager {
 });
 }
 
-public Dictionary getDictionary(String resourcePath) throws IOException 
{
+public Dictionary getDictionary(String resourcePath) throws 
IOException {
 DictionaryInfo dictInfo = getDictionaryInfo(resourcePath);
 return dictInfo == null ? null : dictInfo.getDictionaryObject();
 }
@@ -130,7 +130,7 @@ public class DictionaryManager {
  * Save the dictionary as it is.
  * More often you should consider using its alternative trySaveNewDict to 
save dict space
  */
-public DictionaryInfo forceSave(Dictionary newDict, DictionaryInfo 
newDictInfo) throws IOException {
+

kylin git commit: KYLIN-2380 refactor DbUnit assertion

2017-01-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 78d7086e5 -> 5f6450d6e


KYLIN-2380 refactor DbUnit assertion


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f6450d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f6450d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f6450d6

Branch: refs/heads/master
Commit: 5f6450d6e77f9f62c3670c20ec4d8aef18bfd2fb
Parents: 78d7086
Author: Li Yang 
Authored: Wed Jan 11 12:10:36 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 12:39:24 2017 +0800

--
 .../apache/kylin/query/HackedDbUnitAssert.java  | 134 ---
 .../apache/kylin/query/ITMassInQueryTest.java   |   9 +-
 .../org/apache/kylin/query/KylinTestBase.java   |  65 +
 3 files changed, 152 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f6450d6/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index c295430..3a21570 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -26,16 +26,34 @@ import org.dbunit.dataset.Columns;
 import org.dbunit.dataset.DataSetException;
 import org.dbunit.dataset.ITable;
 import org.dbunit.dataset.ITableMetaData;
+import org.dbunit.dataset.datatype.BigIntegerDataType;
 import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.IntegerDataType;
+import org.dbunit.dataset.datatype.UnknownDataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * dirty hack to support checking result of SQL with limit
+ * A copy of DbUnitAssert, in which a few hacks applied
+ * 
+ * - tolerate some column type difference, like INT vs. BIGINT
+ * - check expected table contains actual table (instead of equals), for sql 
with limit
  */
 public class HackedDbUnitAssert extends DbUnitAssert {
 private static final Logger logger = 
LoggerFactory.getLogger(HackedDbUnitAssert.class);
 
+private boolean hackCheckContains;
+private boolean hackIgnoreIntBigIntMismatch;
+
+public void hackCheckContains() {
+hackCheckContains = true;
+}
+
+public void hackIgnoreIntBigIntMismatch() {
+hackIgnoreIntBigIntMismatch = true;
+}
+
+// THIS METHOD IS MOSTLY COPIED FROM DbUnitAssert. CHANGES ARE LEAD BY 
hackXXX CONDITION CHECKS.
 public void assertEquals(ITable expectedTable, ITable actualTable, 
FailureHandler failureHandler) throws DatabaseUnitException {
 logger.trace("assertEquals(expectedTable, actualTable, failureHandler) 
- start");
 logger.debug("assertEquals: expectedTable={}", expectedTable);
@@ -57,25 +75,21 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 ITableMetaData actualMetaData = actualTable.getTableMetaData();
 String expectedTableName = expectedMetaData.getTableName();
 
-//// Verify row count
-//int expectedRowsCount = expectedTable.getRowCount();
-//int actualRowsCount = actualTable.getRowCount();
-//if (expectedRowsCount != actualRowsCount) {
-//String msg = "row count (table=" + expectedTableName + 
")";
-//Error error =
-//failureHandler.createFailure(msg, String
-//.valueOf(expectedRowsCount), String
-//.valueOf(actualRowsCount));
-//logger.error(error.toString());
-//throw error;
-//}
-
-// if both tables are empty, it is not necessary to compare columns, as
-// such
-// comparison
-// can fail if column metadata is different (which could occurs when
-// comparing empty tables)
-if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
+// Verify row count
+int expectedRowsCount = expectedTable.getRowCount();
+int actualRowsCount = actualTable.getRowCount();
+if (!hackCheckContains) {
+if (expectedRowsCount != actualRowsCount) {
+String msg = "row count (table=" + expectedTableName + ")";
+Error error = failureHandler.createFailure(msg, 
String.valueOf(expectedRowsCount), String.valueOf(actualRowsCount));
+logger.error(error.toString());
+throw error;
+}
+}
+
+// if both tables are empty, it is not necessary to compare columns,

kylin git commit: KYLIN-2380 refactor DbUnit assertion

2017-01-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang22 3a3ee3c1c -> 2e0881596


KYLIN-2380 refactor DbUnit assertion


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e088159
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e088159
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e088159

Branch: refs/heads/yang22
Commit: 2e0881596e251c7e517748c1468d459ec86210d3
Parents: 3a3ee3c
Author: Li Yang 
Authored: Wed Jan 11 12:10:36 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 12:10:36 2017 +0800

--
 .../apache/kylin/query/HackedDbUnitAssert.java  | 125 ---
 .../apache/kylin/query/ITMassInQueryTest.java   |   9 +-
 .../org/apache/kylin/query/KylinTestBase.java   |  60 +
 3 files changed, 148 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e088159/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index 338f698..3a21570 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -26,16 +26,34 @@ import org.dbunit.dataset.Columns;
 import org.dbunit.dataset.DataSetException;
 import org.dbunit.dataset.ITable;
 import org.dbunit.dataset.ITableMetaData;
+import org.dbunit.dataset.datatype.BigIntegerDataType;
 import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.IntegerDataType;
+import org.dbunit.dataset.datatype.UnknownDataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * dirty hack to support checking result of SQL with limit
+ * A copy of DbUnitAssert, in which a few hacks applied
+ * 
+ * - tolerate some column type difference, like INT vs. BIGINT
+ * - check expected table contains actual table (instead of equals), for sql 
with limit
  */
 public class HackedDbUnitAssert extends DbUnitAssert {
 private static final Logger logger = 
LoggerFactory.getLogger(HackedDbUnitAssert.class);
 
+private boolean hackCheckContains;
+private boolean hackIgnoreIntBigIntMismatch;
+
+public void hackCheckContains() {
+hackCheckContains = true;
+}
+
+public void hackIgnoreIntBigIntMismatch() {
+hackIgnoreIntBigIntMismatch = true;
+}
+
+// THIS METHOD IS MOSTLY COPIED FROM DbUnitAssert. CHANGES ARE LEAD BY 
hackXXX CONDITION CHECKS.
 public void assertEquals(ITable expectedTable, ITable actualTable, 
FailureHandler failureHandler) throws DatabaseUnitException {
 logger.trace("assertEquals(expectedTable, actualTable, failureHandler) 
- start");
 logger.debug("assertEquals: expectedTable={}", expectedTable);
@@ -57,22 +75,21 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 ITableMetaData actualMetaData = actualTable.getTableMetaData();
 String expectedTableName = expectedMetaData.getTableName();
 
-//// Verify row count
-//int expectedRowsCount = expectedTable.getRowCount();
-//int actualRowsCount = actualTable.getRowCount();
-//if (expectedRowsCount != actualRowsCount) {
-//String msg = "row count (table=" + expectedTableName + 
")";
-//Error error =
-//failureHandler.createFailure(msg, String
-//.valueOf(expectedRowsCount), String
-//.valueOf(actualRowsCount));
-//logger.error(error.toString());
-//throw error;
-//}
+// Verify row count
+int expectedRowsCount = expectedTable.getRowCount();
+int actualRowsCount = actualTable.getRowCount();
+if (!hackCheckContains) {
+if (expectedRowsCount != actualRowsCount) {
+String msg = "row count (table=" + expectedTableName + ")";
+Error error = failureHandler.createFailure(msg, 
String.valueOf(expectedRowsCount), String.valueOf(actualRowsCount));
+logger.error(error.toString());
+throw error;
+}
+}
 
 // if both tables are empty, it is not necessary to compare columns, 
as such comparison
 // can fail if column metadata is different (which could occurs when 
comparing empty tables)
-if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
+if (expectedRowsCount == 0 && actualRowsCount == 0) {
 logger.debug("Tables are empty, hence

Jenkins build is back to stable : Kylin-Master-JDK-1.7 #87

2017-01-10 Thread Apache Jenkins Server

See

Jenkins build became unstable: Kylin-Master-JDK-1.7 #86

2017-01-10 Thread Apache Jenkins Server

See

kylin git commit: rollback ITEM_COUNT to INT type

2017-01-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 521dfffae -> 78d7086e5


rollback ITEM_COUNT to INT type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78d7086e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78d7086e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78d7086e

Branch: refs/heads/master
Commit: 78d7086e51c9311ab25b17f05206766d8a279afa
Parents: 521dfff
Author: Li Yang 
Authored: Wed Jan 11 09:40:55 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 11 09:40:55 2017 +0800

--
 .../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/78d7086e/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index df53de9..2b33e66 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -48,7 +48,7 @@
   }, {
 "id" : "10",
 "name" : "ITEM_COUNT",
-"datatype" : "bigint",
+"datatype" : "int",
 "data_gen" : "RAND"
   }, {
 "id" : "11",

kylin git commit: rollback ITEM_COUNT to INT type

2017-01-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang22 fb424042e -> 3a3ee3c1c


rollback ITEM_COUNT to INT type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a3ee3c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a3ee3c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a3ee3c1

Branch: refs/heads/yang22
Commit: 3a3ee3c1cae56d4fed6948d64de23178c1a2cd1e
Parents: fb42404
Author: Yang Li 
Authored: Wed Jan 11 07:15:24 2017 +0800
Committer: Yang Li 
Committed: Wed Jan 11 07:15:24 2017 +0800

--
 .../localmeta/table/DEFAULT.TEST_KYLIN_FACT.json  | 2 +-
 .../test/java/org/apache/kylin/query/HackedDbUnitAssert.java  | 7 ++-
 2 files changed, 3 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3ee3c1/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 53dc96b..fdfb322 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -33,7 +33,7 @@
   }, {
 "id" : "8",
 "name" : "ITEM_COUNT",
-"datatype" : "bigint"
+"datatype" : "int"
   }, {
 "id" : "9",
 "name" : "SELLER_ID",

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3ee3c1/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java 
b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index c295430..338f698 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -70,11 +70,8 @@ public class HackedDbUnitAssert extends DbUnitAssert {
 //throw error;
 //}
 
-// if both tables are empty, it is not necessary to compare columns, as
-// such
-// comparison
-// can fail if column metadata is different (which could occurs when
-// comparing empty tables)
+// if both tables are empty, it is not necessary to compare columns, 
as such comparison
+// can fail if column metadata is different (which could occurs when 
comparing empty tables)
 if (expectedTable.getRowCount() == 0 &&  actualTable.getRowCount() == 
0) {
 logger.debug("Tables are empty, hence equals.");
 return;

kylin git commit: split IT

2017-01-10 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/calcite-11 88cba3d15 -> 1f453cf10


split IT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1f453cf1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1f453cf1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1f453cf1

Branch: refs/heads/calcite-11
Commit: 1f453cf108bdd7dfe601d11d724b23708bd6b009
Parents: 88cba3d
Author: Billy Liu 
Authored: Wed Jan 11 00:50:23 2017 +0800
Committer: Billy Liu 
Committed: Wed Jan 11 00:50:23 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |  3 ++-
 .../test_case_data/sandbox/kylin.properties |  2 +-
 jdbc/pom.xml| 14 +
 kylin-it/pom.xml| 21 +++-
 pom.xml |  4 +---
 server/pom.xml  |  6 --
 6 files changed, 12 insertions(+), 38 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1f453cf1/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
index f7780dd..d0c352c 100755
--- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -22,4 +22,5 @@ cd ${dir}
 cd ..
 
 mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee 
mvnverify.log
+mvn verify -Psandbox-build -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 
| tee mvnverify-build.log
+mvn verify -Psandbox-query -pl kylin-it -fae 2>&1 | tee mvnverify-query.log

http://git-wip-us.apache.org/repos/asf/kylin/blob/1f453cf1/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 1724619..08caa7e 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -25,7 +25,7 @@ kylin.server.mode=all
 kylin.storage.hbase.owner-tag=who...@kylin.apache.org
 
 # List of web servers in use, this enables one web server instance to sync up 
with other servers.
-kylin.server.cluster-servers=localhost:7070
+#kylin.server.cluster-servers=localhost:7070
 
 # Display timezone on UI,format like[GMT+N or GMT-N]
 kylin.web.timezone=GMT+8

http://git-wip-us.apache.org/repos/asf/kylin/blob/1f453cf1/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 0a70949..993822f 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -44,12 +44,7 @@
 
 
 org.apache.calcite.avatica
-avatica
-
-
-com.google.protobuf
-protobuf-java
-${avatica.protobuf.version}
+avatica-core
 
 
 junit
@@ -72,13 +67,6 @@
 
 
 
false
-
-
-
-
com.fasterxml.jackson.core:*
-
-
 
 
 org.apache.calcite

http://git-wip-us.apache.org/repos/asf/kylin/blob/1f453cf1/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 55b9f98..c5242b4 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -57,16 +57,6 @@
 
 
 org.apache.kylin
-kylin-jdbc
-
-
-com.google.protobuf
-protobuf-java
-
-
-
-
-org.apache.kylin
 kylin-tool
 
 
@@ -251,9 +241,8 @@
 sandbox-query
 
 
-com.google.protobuf
-protobuf-java
-${avatica.protobuf.version}
+org.apache.kylin
+kylin-jdbc
 
 
 
@@ -288,6 +277,10 @@
 log4j.configuration
 
file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
 
+
+

kylin git commit: add multipe profile to IT

2017-01-10 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/calcite-11 2b72f7433 -> 88cba3d15


add multipe profile to IT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/88cba3d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/88cba3d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/88cba3d1

Branch: refs/heads/calcite-11
Commit: 88cba3d15dfd036c36937d18371daf55e3410447
Parents: 2b72f74
Author: Billy Liu 
Authored: Tue Jan 10 22:01:00 2017 +0800
Committer: Billy Liu 
Committed: Tue Jan 10 22:01:00 2017 +0800

--
 kylin-it/pom.xml | 30 --
 1 file changed, 24 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/88cba3d1/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 94cdb7e..55b9f98 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -55,12 +55,16 @@
 org.apache.kylin
 kylin-source-hive
 
-
 
 org.apache.kylin
 kylin-tool
@@ -244,10 +248,14 @@
 
 
 
-sandbox
-
-true
-
+sandbox-query
+
+
+com.google.protobuf
+protobuf-java
+${avatica.protobuf.version}
+
+
 
 
 
@@ -284,6 +292,16 @@
 -Xms1G -Xmx2G -XX:PermSize=128M 
-XX:MaxPermSize=512M
 
 
+
+
+
+
+sandbox
+
+true
+
+
+
 
 org.codehaus.mojo
 exec-maven-plugin

kylin git commit: KYLIN-2376 upgrade checkstyle plugin

2017-01-10 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/master c84b46bd5 -> 521dfffae


KYLIN-2376 upgrade checkstyle plugin

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/521dfffa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/521dfffa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/521dfffa

Branch: refs/heads/master
Commit: 521dfffae15d0c85885ec816a4fc40907bfb7645
Parents: c84b46b
Author: etherge 
Authored: Tue Jan 10 19:09:58 2017 +0800
Committer: Billy Liu 
Committed: Tue Jan 10 20:53:57 2017 +0800

--
 pom.xml | 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/521dfffa/pom.xml
--
diff --git a/pom.xml b/pom.xml
index fcb5ce3..e391103 100644
--- a/pom.xml
+++ b/pom.xml
@@ -907,6 +907,13 @@
 org.apache.maven.plugins
 maven-checkstyle-plugin
 2.17
+
+
+com.puppycrawl.tools
+checkstyle
+6.19
+
+
 
 
 check-style

[2/6] kylin git commit: rollback ITEM_COUNT to INT type

[1/6] kylin git commit: alter query test case to fix CI [Forced Update!]

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[6/6] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[5/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[1/6] kylin git commit: alter query test case to fix CI [Forced Update!]

[2/6] kylin git commit: rollback ITEM_COUNT to INT type

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[3/6] kylin git commit: KYLIN-2380 refactor DbUnit assertion

[5/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[3/5] kylin git commit: KYLIN-2380 refactor DbUnit assertion

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[2/5] kylin git commit: rollback ITEM_COUNT to INT type

[1/5] kylin git commit: alter query test case to fix CI [Forced Update!]

kylin git commit: minor, change Dictionary to Dictionary in DictionaryManager and DictionaryInfo

kylin git commit: KYLIN-2380 refactor DbUnit assertion

kylin git commit: KYLIN-2380 refactor DbUnit assertion

Jenkins build is back to stable : Kylin-Master-JDK-1.7 #87

Jenkins build became unstable: Kylin-Master-JDK-1.7 #86

kylin git commit: rollback ITEM_COUNT to INT type

kylin git commit: rollback ITEM_COUNT to INT type

kylin git commit: split IT

kylin git commit: add multipe profile to IT

kylin git commit: KYLIN-2376 upgrade checkstyle plugin

24 matches

Site Navigation

Mail list logo

Footer information