[2/4] kylin git commit: KYLIN-2192 More Robust Global Dictionary

[gaodayue]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
--
diff --git 
a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
 
b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
index caa8754..949599d 100644
--- 
a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
+++ 
b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
@@ -1,402 +1,402 @@
-2013-03-31,48028,0,\N,\N,\N,Auction,12,184.21,1001,1,Ebay
-2013-11-12,164262,0,\N,\N,\N,Others,5,172.03,1002,1,Ebay
-2013-04-06,82494,15,BookMagazines,NULL,Comic 
Books,Auction,14,66.6,1003,1,Ebay
-2013-05-17,66767,15,Home & Garden,NULL,Dogs,Auction,12,92.98,1004,1,Ebay
-2013-05-20,152801,0,Jewelry & 
Watches,NULL,Earrings,FP-GTC,5,132.33,1005,1,Ebay
-2013-06-16,43398,0,Home & Garden,NULL,Cheese & 
Crackers,FP-GTC,13,7.12,1006,1,Ebay
-2013-06-14,95173,0,Health & Beauty,Bath & Body,Bath Sets & 
Kits,Auction,14,204.28,1007,1,Ebay
-2013-03-22,158666,15,ToyHobbies,Action Figures,Anime & 
Manga,Auction,13,35.72,1008,1,Ebay
-2013-03-10,12688,0,eBay Premier,Books & Manuscripts,Books: 
Other,Auction,12,4.13,1009,1,Ebay
-2013-11-01,103324,15,ClothinShoeAccessories,Women's Shoes,Mixed 
Items,FP-GTC,5,27.48,1010,1,Ebay
-2013-06-16,108782,15,Vehicle Parts & Accessories,CaTruck Parts,Car Care & 
Cleaning,FP-GTC,14,9.26,1011,1,Ebay
-2013-09-12,80287,0,Computers/Tablets & Networking,Software,Office & 
Business,Auction,12,3.18,1012,1,Ebay
-2013-09-28,140746,100,eBay Motors,Parts & Accessories,Vintage Car & Truck 
Parts,Others,13,3.18,1013,1,Ebay
-2013-06-15,87118,0,Sporting Goods,Outdoor 
Sports,Paintball,ABIN,14,377.94,1014,1,Ebay
-2013-03-14,25147,0,Sports MeCards & Fan Shop,Fan Apparel & 
Souvenirs,Baseball-MLB,Auction,12,146.33,1015,1,Ebay
-2013-09-01,170302,15,Crafts,Embroidery,Design 
CDs,FP-GTC,5,51.23,1016,1,Ebay
-2013-05-29,53064,0,Business & Industrial,Heavy Equipment,Antique & Vintage 
Farm Equip,FP-non GTC,13,72.65,1017,1,Ebay
-2013-05-31,132939,0,Jewelry & Watches,Fashion 
Jewelry,Other,Auction,13,66.6,1018,1,Ebay
-2013-03-18,113593,15,Phones,Mobile Phones,Mobile 
Phones,Auction,12,9.26,1019,1,Ebay
-2013-07-19,34273,100,eBay Motors,Parts & 
Accessories,Motorcycle,Auction,14,583.44,1020,1,Ebay
-2013-06-23,106340,15,Home & Garden,Gardening,Hand 
Tools,FP-GTC,14,638.72,1021,1,Ebay
-2013-05-20,150265,15,Baby,Baby Clothing,Boys,FP-GTC,14,4.54,1022,1,Ebay
-2013-05-17,24760,0,Sports MeCards & Fan Shop,Fan Apparel & 
Souvenirs,Hockey-NHL,FP-GTC,12,319.79,1023,1,Ebay
-2013-03-11,37831,0,Collectibles,Advertising,Merchandise & 
Memorabilia,Auction,12,20.35,1024,1,Ebay
-2013-01-30,1120,3,Books,First Editions,Other,FP-non 
GTC,5,223.63,1025,1,Ebay
-2013-01-26,43972,100,eBay Motors,Parts & Accessories,ATV 
Parts,FP-GTC,13,204.28,1026,1,Ebay
-2013-03-22,166013,15,Computers,Computer Components & Parts,Video Capture & TV 
Tuner Cards,Auction,14,5.48,1027,1,Ebay
-2013-07-23,15568,15,Baby,Baby Clothing,Unisex,Auction,14,27.48,1028,1,Ebay
-2013-07-27,103178,15,ClothinShoeAccessories,Women's Bags,Women's 
Bags,FP-GTC,5,21.72,1029,1,Ebay
-2013-10-29,2023,0,Sporting Goods,Team 
Sports,Basketball,ABIN,12,3.18,1030,1,Ebay
-2013-10-08,94847,0,Consumer Electronics,Vehicle Electronics & GPS,Car 
Video,FP-GTC,11,491.32,1031,1,Ebay
-2013-04-26,15868,0,Real Estate,Land,Land,Auction,14,448.8,1032,1,Ebay
-2013-01-01,32876,0,Home & Garden,Home Improvement,Plumbing & 
Fixtures,Auction,13,415.73,1033,1,Ebay
-2013-01-15,62179,0,ClothinShoes & Accessories,Women's Clothing,Athletic 
Apparel,Auction,13,377.94,1034,1,Ebay
-2013-05-27,33038,15,Musical Instruments,Instruments,Guitars 
(Electric),FP-GTC,14,146.33,1035,1,Ebay
-2013-11-11,156614,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: 
RacinNASCAR,FP-GTC,5,7.12,1036,1,Ebay
-2013-03-08,106246,0,Health & Beauty,Hair Care & Styling,Shampoo & 
Conditioning,Auction,13,42.99,1037,1,Ebay
-2013-03-25,20865,0,ClothinShoes & Accessories,Men's Clothing,Athletic 
Apparel,Auction,13,12.85,1038,1,Ebay
-2013-08-20,15115,0,Video Games & Consoles,Video Games,Video 
Games,FP-GTC,13,55.89,1039,1,Ebay
-2013-05-17,3838,0,Jewelry & Watches,Fashion Jewelry,Charms & Charm 
Bracelets,FP-GTC,14,73.26,1040,1,Ebay
-2013-06-05,759,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & 
Vans,Auction,11,112.56,1041,1,Ebay
-2013-10-08,61323,0,Consumer Electronics,TVideo & Home Audio,TVideo & Audio 
Accessories,FP-non GTC,11,3.49,1042,1,Ebay
-2013-08-14,121153,0,Baby,Nursery Decor,Night 
Lights,Auction,13,184.21,1043,1,Ebay
-2013-08-14,88750,0,Consumer Electronics,Vehicle Electronics & GPS,Radar & 
Laser Detectors,Auction,13,157.14,1044,1,Ebay

[4/4] kylin git commit: KYLIN-2192 More Robust Global Dictionary

[gaodayue]

KYLIN-2192 More Robust Global Dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a0ee798
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a0ee798
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a0ee798

Branch: refs/heads/master
Commit: 4a0ee7989d5f8272592b980fce3f5716ca40d4c1
Parents: e562aaf
Author: sunyerui 
Authored: Mon Nov 21 21:26:34 2016 +0800
Committer: gaodayue 
Committed: Fri Dec 2 13:33:59 2016 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   |  12 +-
 .../apache/kylin/common/KylinConfigBase.java|   8 +
 .../apache/kylin/common/util/Dictionary.java|   2 +-
 .../model/validation/rule/DictionaryRule.java   |  78 +-
 .../validation/rule/DictionaryRuleTest.java |  28 +-
 .../apache/kylin/dict/AppendTrieDictionary.java | 285 +--
 .../kylin/dict/AppendTrieDictionaryChecker.java | 102 +++
 .../org/apache/kylin/dict/CachedTreeMap.java| 260 +++---
 .../kylin/dict/GlobalDictionaryBuilder.java |  36 +-
 .../kylin/dict/AppendTrieDictionaryTest.java| 150 +++-
 .../apache/kylin/dict/CachedTreeMapTest.java| 320 +---
 .../kylin/measure/bitmap/BitmapCounterTest.java |   6 +-
 ...t_kylin_cube_without_slr_left_join_desc.json |  16 +-
 .../localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv  | 804 +--
 .../flatten_data_for_without_slr_left_join.csv  | 804 +--
 .../test_kylin_inner_join_model_desc.json   |   3 +-
 .../test_kylin_inner_join_view_model_desc.json  |   3 +-
 .../test_kylin_left_join_model_desc.json|   3 +-
 .../test_kylin_left_join_view_model_desc.json   |   3 +-
 .../table/DEFAULT.TEST_KYLIN_FACT.json  |   8 +-
 .../source/hive/ITHiveTableReaderTest.java  |   2 +-
 .../query/sql_distinct_precisely/query00.sql|   2 +-
 .../query/sql_distinct_precisely/query01.sql|   2 +-
 .../query/sql_distinct_precisely/query02.sql|   2 +-
 .../query/sql_distinct_precisely/query03.sql|   3 +-
 .../query/sql_distinct_precisely/query04.sql|   3 +-
 .../query/sql_distinct_precisely/query05.sql|  25 -
 .../query/sql_distinct_precisely/query06.sql|  26 -
 .../query/sql_distinct_precisely/query07.sql|  24 -
 29 files changed, 1737 insertions(+), 1283 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 8068fd1..677b713 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -403,13 +403,13 @@ public class FactTableGenerator {
 }
 
 private String createRandomCell(ColumnDesc cDesc) {
-String type = cDesc.getTypeName();
-String s = type.toLowerCase();
-if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
+DataType type =cDesc.getType();
+String s = type.getName();
+if (s.equals("char") || s.equals("varchar")) {
 StringBuilder sb = new StringBuilder();
-for (int i = 0; i < 2; i++) {
-sb.append((char) ('a' + r.nextInt(10)));// there are 10*10
-// possible strings
+int len = Math.min(type.getPrecision(), 3);
+for (int i = 0; i < len; i++) {
+sb.append((char) ('a' + r.nextInt(10)));  // cardinality at 
most 10x10x10
 }
 return sb.toString();
 } else if (s.equals("bigint") || s.equals("int") || 
s.equals("tinyint") || s.equals("smallint")) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3c10826..f46c185 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -232,6 +232,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "1000"));
 }
 
+public int getAppendDictMaxVersions() {
+return 
Integer.parseInt(getOptional("kylin.dictionary.append-max-versions", "3"));
+}
+
+public int getAppendDictVersionTTL() {
+return 

[1/4] kylin git commit: KYLIN-2192 More Robust Global Dictionary

[gaodayue]

Repository: kylin
Updated Branches:
  refs/heads/master e562aafee -> 4a0ee7989


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
index a3948c3..0c163a6 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
@@ -19,6 +19,6 @@
 select lstg_format_name, cal_dt,
  sum(price) as GMV,
  count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
+ count(distinct user_id) as user_count
  from test_kylin_fact
  group by lstg_format_name, cal_dt

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
index e8579ef..62142a2 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
@@ -19,7 +19,7 @@
 select lstg_format_name,
  sum(price) as GMV,
  count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
+ count(distinct user_id) as user_count
  from test_kylin_fact
  where lstg_format_name='FP-GTC'
  group by lstg_format_name

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
index 48f49e9..88ce532 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
@@ -19,7 +19,7 @@
 select lstg_format_name,
  sum(price) as GMV,
  count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
+ count(distinct user_id) as user_count
  from test_kylin_fact
  where lstg_format_name='FP-GTC'
  group by lstg_format_name

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
index dbc2fac..7f739e7 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
@@ -17,7 +17,8 @@
 --
 
 select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count
+ , count(1) as TRANS_CNT
+ , count(distinct user_id) as user_count
  , count(distinct site_name) as site_count
  from test_kylin_fact
  inner JOIN edw.test_cal_dt as test_cal_dt

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
index 69006ce..13de5a7 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
@@ -17,7 +17,8 @@
 --
 
 select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count
+ , count(1) as TRANS_CNT
+ , count(distinct user_id) as user_count
  , count(distinct site_name) as site_count
  from test_kylin_fact
  inner JOIN edw.test_cal_dt as test_cal_dt

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
deleted file mode 100644
index dea09f7..000
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
+++ /dev/null
@@ -1,25 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 

[3/4] kylin git commit: KYLIN-2192 More Robust Global Dictionary

[gaodayue]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java 
b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
index df64f3f..381e0b1 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
@@ -17,11 +17,12 @@
 */
 package org.apache.kylin.dict;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.*;
@@ -91,39 +92,26 @@ public class CachedTreeMapTest {
 public static class CachedFileFilter implements FileFilter {
 @Override
 public boolean accept(File pathname) {
-return pathname.getName().startsWith("cached_");
+return pathname.getName().startsWith(CachedTreeMap.CACHED_PREFIX);
 }
 }
 
-public static final String baseDir = "/tmp/kylin_cachedtreemap_test/";
-public static final String backupDir = 
"/tmp/kylin_cachedtreemap_test.bak/";
-public static final String tmpDir = "/tmp/kylin_cachedtreemap_test.tmp/";
-
-private static void cleanup() {
-File dir = new File(baseDir);
-if (dir.exists()) {
-for (File f : dir.listFiles()) {
-f.delete();
-}
-dir.delete();
-}
-
-dir = new File(tmpDir);
-if (dir.exists()) {
-for (File f : dir.listFiles()) {
-f.delete();
-}
-dir.delete();
+public static class VersionFilter implements FileFilter {
+@Override
+public boolean accept(File pathname) {
+return pathname.getName().startsWith(CachedTreeMap.VERSION_PREFIX);
 }
+}
 
-dir = new File(backupDir);
-if (dir.exists()) {
-for (File f : dir.listFiles()) {
-f.delete();
-}
-dir.delete();
-}
+public static final String baseDir = "/tmp/kylin_cachedtreemap_test/";
+public static final String workingDir = 
"/tmp/kylin_cachedtreemap_test/working";
 
+private static void cleanup() {
+Configuration conf = new Configuration();
+Path basePath = new Path(baseDir);
+try {
+FileSystem.get(basePath.toUri(), conf).delete(basePath, true);
+} catch (IOException e) {}
 VALUE_WRITE_ERROR_TOGGLE = false;
 }
 
@@ -139,154 +127,240 @@ public class CachedTreeMapTest {
 
 @Test
 public void testCachedTreeMap() throws IOException {
-CachedTreeMap map = 
CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
-
.persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
+CachedTreeMap map = createMutableMap();
 map.put(Key.of(1), Value.of("a"));
 map.put(Key.of(2), Value.of("b"));
 map.put(Key.of(3), Value.of("c"));
 map.put(Key.of(4), Value.of("d"));
 map.put(Key.of(5), Value.of("e"));
 
-File dir = new File(tmpDir);
+File dir = new File(workingDir);
 assertEquals(3, dir.listFiles(new CachedFileFilter()).length);
 
-DataOutputStream out = new DataOutputStream(new 
FileOutputStream(tmpDir+"/.index"));
-map.write(out);
-out.flush();
-out.close();
-map.commit(false);
+flushAndCommit(map, true, true, false);
+assertFalse(new File(workingDir).exists());
 
-dir = new File(baseDir);
+dir = new File(map.getLatestVersion());
 assertEquals(5, dir.listFiles(new CachedFileFilter()).length);
 
-DataInputStream in = new DataInputStream(new 
FileInputStream(baseDir+".index"));
-CachedTreeMap map2 = 
CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
-
.persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
-map2.readFields(in);
+CachedTreeMap map2 = createImmutableMap();
 assertEquals(5, map2.size());
 assertEquals("b", ((Value)map2.get(Key.of(2))).valueStr);
 
 try {
 map2.put(Key.of(6), Value.of("f"));
 fail("Should be error when put value into immutable map");
-} catch (AssertionError error) {
+} catch (AssertionError error) {}
+}
+
+@Test
+public void testMultiVersions() throws IOException, InterruptedException {
+CachedTreeMap map = createMutableMap();
+Thread.sleep(3000);
+map.put(Key.of(1), Value.of("a"));
+map.put(Key.of(2), 

kylin git commit: KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 3091f06a6 -> e562aafee


KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e562aafe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e562aafe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e562aafe

Branch: refs/heads/master
Commit: e562aafee10c8bbd125937503d42b958c1843aad
Parents: 3091f06
Author: Hongbin Ma 
Authored: Fri Dec 2 13:33:02 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Dec 2 13:33:18 2016 +0800

--
 .../kylin/common/debug/BackdoorToggles.java   | 18 ++
 .../java/org/apache/kylin/jdbc/IRemoteClient.java |  3 ++-
 .../java/org/apache/kylin/jdbc/KylinClient.java   |  7 ---
 .../org/apache/kylin/jdbc/KylinResultSet.java |  9 -
 .../org/apache/kylin/jdbc/json/QueryRequest.java  | 12 
 .../java/org/apache/kylin/jdbc/DummyClient.java   |  3 ++-
 .../apache/kylin/rest/service/QueryService.java   | 11 ++-
 7 files changed, 56 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index f7c90aa..28f7697 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -79,6 +79,14 @@ public class BackdoorToggles {
 }
 }
 
+public static Integer getStatementMaxRows() {
+String v = getString(ATTR_STATEMENT_MAX_ROWS);
+if (v == null)
+return null;
+else
+return Integer.valueOf(v);
+}
+
 private static String getString(String key) {
 Map toggles = _backdoorToggles.get();
 if (toggles == null) {
@@ -183,4 +191,14 @@ public class BackdoorToggles {
  */
 public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = 
"DEBUG_TOGGLE_SHARD_ASSIGNMENT";
 
+// properties on statement may go with this "channel" too
+/**
+ * set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "ATTR_STATEMENT_MAX_ROWS": "10"
+ }
+ */
+public final static String ATTR_STATEMENT_MAX_ROWS = 
"ATTR_STATEMENT_MAX_ROWS";
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
index b6a13e5..dfd8d76 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.calcite.avatica.AvaticaParameter;
 import org.apache.calcite.avatica.ColumnMetaData;
@@ -51,6 +52,6 @@ public interface IRemoteClient extends Closeable {
 /**
  * Execute query remotely and get back result.
  */
-public QueryResult executeQuery(String sql, List params, 
List paramValues) throws IOException;
+public QueryResult executeQuery(String sql, List params, 
List paramValues, Map queryToggles) throws IOException;
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index 2d06a92..86c3a5b 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -321,9 +321,9 @@ public class KylinClient implements IRemoteClient {
 }
 
 @Override
-public QueryResult executeQuery(String sql, List params, 
List paramValues) throws IOException {
+public QueryResult executeQuery(String sql, List params, 
List paramValues, Map queryToggles) throws IOException {
 
-SQLResponseStub queryResp = executeKylinQuery(sql, 
convertParameters(params, paramValues));
+SQLResponseStub queryResp = executeKylinQuery(sql, 

kylin git commit: KYLIN-2213 add test query

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 9fc8f5e20 -> 3091f06a6


KYLIN-2213 add test query


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3091f06a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3091f06a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3091f06a

Branch: refs/heads/master
Commit: 3091f06a68e0b97b35639cbb65acbf532c114ca4
Parents: 9fc8f5e
Author: Li Yang 
Authored: Fri Dec 2 13:21:54 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 13:21:54 2016 +0800

--
 .../test/resources/query/sql_like/query20.sql   | 31 
 1 file changed, 31 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3091f06a/kylin-it/src/test/resources/query/sql_like/query20.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_like/query20.sql 
b/kylin-it/src/test/resources/query/sql_like/query20.sql
new file mode 100644
index 000..6101cb0
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query20.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select lstg_format_name as lstg_format_name, count(*) as cnt 
+ 
+ from test_kylin_fact 
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ 
+ 
+where lstg_format_name not like '%BIN%'
+group by lstg_format_name
\ No newline at end of file

kylin git commit: KYLIN-2180 minor, get owenr project in CubeDesc.init()

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 1a5295bad -> 9fc8f5e20


KYLIN-2180 minor, get owenr project in CubeDesc.init()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9fc8f5e2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9fc8f5e2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9fc8f5e2

Branch: refs/heads/master
Commit: 9fc8f5e207b5111fffaec3335efe0c8947353cfa
Parents: 1a5295b
Author: Li Yang 
Authored: Fri Dec 2 11:39:34 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 11:39:34 2016 +0800

--
 .../main/java/org/apache/kylin/cube/model/CubeDesc.java  | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9fc8f5e2/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 327ce57..853571c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -64,6 +64,9 @@ import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -519,11 +522,16 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public void init(KylinConfig config) {
 this.errors.clear();
-this.config = KylinConfigExt.createInstance(config, 
overrideKylinProps);
 
 checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank");
 checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc(%s) has 
blank modelName", name);
 
+// note CubeDesc.name == CubeInstance.name
+List ownerPrj = 
ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name);
+logger.info("CubeDesc '" + name + "' is owned by " + ownerPrj);
+
+this.config = KylinConfigExt.createInstance(config, 
overrideKylinProps);
+
 this.model = 
MetadataManager.getInstance(config).getDataModelDesc(modelName);
 checkNotNull(this.model, "DateModelDesc(%s) not found", modelName);
 
@@ -823,6 +831,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 return col;
 }
 
+@SuppressWarnings("deprecation")
 private void initMeasureColumns() {
 if (measures == null || measures.isEmpty()) {
 return;

kylin git commit: minor, rollback ITKylinQueryTest

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1875 48253d1ff -> 12230f4e3


minor, rollback ITKylinQueryTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/12230f4e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/12230f4e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/12230f4e

Branch: refs/heads/KYLIN-1875
Commit: 12230f4e3b6a1557137e38b2e3c8c3b01288c248
Parents: 48253d1
Author: Yang Li 
Authored: Fri Dec 2 06:32:25 2016 +0800
Committer: Yang Li 
Committed: Fri Dec 2 06:32:25 2016 +0800

--
 .../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/12230f4e/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 6d91753..90324b5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -48,7 +48,7 @@ import org.junit.rules.ExpectedException;
 
 import com.google.common.collect.Maps;
 
-//@Ignore("KylinQueryTest is contained by ITCombinationTest")
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
 public class ITKylinQueryTest extends KylinTestBase {
 
 @Rule

kylin git commit: KYLIN-1875 fix PK-FK derive from each other

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1875 a9100bfc6 -> 48253d1ff


KYLIN-1875 fix PK-FK derive from each other


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/48253d1f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/48253d1f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/48253d1f

Branch: refs/heads/KYLIN-1875
Commit: 48253d1ff03e137a8568b8ca880c45b5b8c524f3
Parents: a9100bf
Author: Yang Li 
Authored: Thu Dec 1 21:42:52 2016 +0800
Committer: Yang Li 
Committed: Thu Dec 1 21:42:52 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  8 +--
 .../org/apache/kylin/cube/model/CubeDesc.java   | 52 +++-
 .../storage/gtrecord/CubeTupleConverter.java|  2 +-
 .../gtrecord/GTCubeStorageQueryBase.java|  2 +-
 .../translate/DerivedFilterTranslator.java  |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  2 +-
 .../query/enumerator/LookupTableEnumerator.java |  2 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  2 +-
 .../hbase/cube/v1/CubeTupleConverter.java   |  2 +-
 9 files changed, 41 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b4422d2..6a25594 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -45,7 +45,6 @@ import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -55,6 +54,7 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -657,10 +657,10 @@ public class CubeManager implements IRealizationProvider {
 cubeMap.removeLocal(cubeName);
 }
 
-public LookupStringTable getLookupTable(CubeSegment cubeSegment, 
DimensionDesc dim) {
+public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc 
join) {
 
-String tableName = dim.getTableRef().getTableIdentity();
-String[] pkCols = dim.getJoin().getPrimaryKey();
+String tableName = join.getPKSide().getTableIdentity();
+String[] pkCols = join.getPrimaryKey();
 String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
 if (snapshotResPath == null)
 throw new IllegalStateException("No snaphot for table '" + 
tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() 
+ "/" + cubeSegment);

http://git-wip-us.apache.org/repos/asf/kylin/blob/48253d1f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index fa4da92..18b7e7a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
@@ -98,20 +99,20 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public static class DeriveInfo {
 public DeriveType type;
-public DimensionDesc dimension;
+public JoinDesc join;
 public TblColRef[] columns;
 public boolean isOneToOne; // only used when ref from derived to host
 
-DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] 
columns, boolean isOneToOne) {
+  

kylin git commit: KYLIN-1875 Support measure on non-root-fact-table

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1875 28ae57b81 -> a9100bfc6


KYLIN-1875 Support measure on non-root-fact-table


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a9100bfc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a9100bfc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a9100bfc

Branch: refs/heads/KYLIN-1875
Commit: a9100bfc6e7d3b601f4721d0bafa198ebb810a98
Parents: 28ae57b
Author: Li Yang 
Authored: Thu Dec 1 18:59:08 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 1 18:59:08 2016 +0800

--
 .../kylin/metadata/project/ProjectL2Cache.java  | 26 +---
 .../template/cube_desc/kylin_sales_cube.json| 18 +++---
 .../kylin/query/relnode/OLAPTableScan.java  |  4 +++
 3 files changed, 30 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a9100bfc/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 82c0de3..14d7843 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
@@ -126,17 +127,17 @@ class ProjectL2Cache {
 return Collections.unmodifiableSet(tableCache.realizations);
 }
 
-public List listEffectiveRewriteMeasures(String project, 
String factTable, boolean onlyRewriteMeasure) {
-Set realizations = getRealizationsByTable(project, 
factTable);
+public List listEffectiveRewriteMeasures(String project, 
String table, boolean onlyRewriteMeasure) {
+Set realizations = getRealizationsByTable(project, 
table);
 List result = Lists.newArrayList();
 for (IRealization r : realizations) {
-if (r.getModel().isFactTable(factTable) && r.isReady()) {
-for (MeasureDesc m : r.getMeasures()) {
-FunctionDesc func = m.getFunction();
-if (onlyRewriteMeasure) {
-if (func.needRewrite())
-result.add(m);
-} else {
+if (!r.isReady())
+continue;
+
+for (MeasureDesc m : r.getMeasures()) {
+FunctionDesc func = m.getFunction();
+if (belongToTable(func, table, r.getModel())) {
+if (!onlyRewriteMeasure || func.needRewrite()) {
 result.add(m);
 }
 }
@@ -145,6 +146,13 @@ class ProjectL2Cache {
 return result;
 }
 
+private boolean belongToTable(FunctionDesc func, String table, 
DataModelDesc model) {
+// measure belong to the first column parameter's table
+List cols = func.getParameter().getColRefs();
+String belongTo = cols.isEmpty() ? 
model.getRootFactTable().getTableIdentity() : cols.get(0).getTable();
+return belongTo.equals(table);
+}
+
 // 

 // build the cache
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/a9100bfc/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 48eef46..2a27305 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -103,26 +103,26 @@
   "returntype" : "decimal(19,4)"
 }
   }, {
-"name" : "GMV_MIN",
+"name" : "BUYER_LEVEL_SUM",
 "function" : {
-  "expression" : "MIN",
+  "expression" : "SUM",
   "parameter" : {
 "type" : "column",
-"value" : "PRICE",
+"value" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
 "next_parameter" : null
   },
-  "returntype" : "decimal(19,4)"
+  "returntype" : "bigint"
 }
   }, {
-"name" : "GMV_MAX",
+"name" : 

kylin git commit: measure encoding length bug

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 58224921d -> 1a5295bad


measure encoding length bug

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a5295ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a5295ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a5295ba

Branch: refs/heads/master
Commit: 1a5295bad21fc18766bb3c7ba6eee59678d42185
Parents: 5822492
Author: luguosheng <550175...@qq.com>
Authored: Thu Dec 1 17:54:35 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 18:17:35 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 4 ++--
 webapp/app/js/controllers/cubeEdit.js   | 2 +-
 webapp/app/js/controllers/cubeMeasures.js   | 8 
 3 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index f2b36fe..8192419 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -47,7 +47,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 var _encoding = item.encoding;
 var _valueLength ;
 var baseKey=item.encoding.replace(/:\d+/,'');
-if(needLengthKeyList.indexOf(baseKey)>=-1){
+if(needLengthKeyList.indexOf(baseKey)!=-1){
   var result=/:(\d+)/.exec(item.encoding);
   _valueLength=result?result[1]:0;
 }
@@ -78,7 +78,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 var version=$scope.getTypeVersion(item.encoding);
 var encodingType=$scope.removeVersion(item.encoding);
 
-if(needLengthKeyList.indexOf(encodingType)>=-1){
+if(needLengthKeyList.indexOf(encodingType)!=-1){
   encoding = encodingType+":"+item.valueLength;
 }else{
   encoding = encodingType;

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 0704a62..85bd4b1 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -78,7 +78,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 $scope.store.supportedEncoding = $scope.cubeConfig.encodings;
   })
   $scope.createFilter=function(type){
- if(type.indexOf("varchar")<=0){
+ if(type.indexOf("varchar")==-1){
return ['fixed_length_hex'];
  }else if(type!="date"){
return ['date'];

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 085338f..18f53c2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -45,7 +45,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 }
   }
   $scope.createFilter=function(type){
-if(type.indexOf("varchar")<=0){
+if(type.indexOf("varchar")==-1){
   return ['fixed_length_hex'];
 }else if(type!="date"){
   return ['date'];
@@ -60,7 +60,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 var encodings =$scope.store.supportedEncoding,filterEncoding=[];
 var filerList=$scope.createFilter(type);
 if($scope.isEdit) {
-  if (name && $scope.newMeasure.function.configuration) {
+  if (name && 
$scope.newMeasure.function.configuration&&$scope.newMeasure.function.configuration['topn.encoding.'
 + name]) {
 var version = 
$scope.newMeasure.function.configuration['topn.encoding_version.' + name] || 1;
 filterEncoding = VdmUtil.getFilterObjectListByOrFilterVal(encodings, 
'value', $scope.newMeasure.function.configuration['topn.encoding.' + 
name].replace(/:\d+/, "") + (version ? "[v" + version + "]" : "[v1]"), 
'suggest', true);
   }else{
@@ -119,7 +119,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 var 
version=$scope.newMeasure.function.configuration['topn.encoding_version.'+_name]||1;
 item=$scope.removeVersion(item);
 var baseKey=item.replace(/:\d+/,'');
-if(needLengthKeyList.indexOf(baseKey)>=-1){
+if(needLengthKeyList.indexOf(baseKey)!=-1){
   

kylin git commit: refine mapper and reducer log

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 28ba1eaea -> 58224921d


refine mapper and reducer log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58224921
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58224921
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58224921

Branch: refs/heads/master
Commit: 58224921d896e4479f5d034d43c044aacaf14200
Parents: 28ba1ea
Author: Hongbin Ma 
Authored: Thu Dec 1 18:15:46 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 18:15:46 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/KylinMapper.java   | 12 +---
 .../java/org/apache/kylin/engine/mr/KylinReducer.java  | 13 ++---
 .../apache/kylin/engine/mr/steps/CuboidReducer.java| 11 ---
 .../kylin/engine/mr/steps/HiveToBaseCuboidMapper.java  |  6 --
 .../kylin/engine/mr/steps/InMemCuboidMapper.java   |  7 +--
 .../kylin/engine/mr/steps/InMemCuboidReducer.java  | 10 +-
 .../apache/kylin/engine/mr/steps/NDCuboidMapper.java   | 10 --
 7 files changed, 33 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index a01f7a2..2b564e9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,6 +31,8 @@ import org.slf4j.LoggerFactory;
 public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
+protected int mapCounter = 0;
+
 protected void bindCurrentConfiguration(Configuration conf) {
 logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 HadoopUtil.setCurrentConfiguration(conf);
@@ -38,6 +41,9 @@ public class KylinMapper 
extends Mapper.Context context) throws IOException, InterruptedException {
 try {
+if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 
0) {
+logger.info("Accepting Mapper Key with ordinal: " + 
mapCounter);
+}
 doMap(key, value, context);
 } catch (IOException ex) { // KYLIN-2170
 logger.error("", ex);
@@ -53,11 +59,11 @@ public class KylinMapper 
extends Mapper.Context context) throws IOException, InterruptedException {
 super.map(key, value, context);
 }
-
+
 @Override
 final protected void cleanup(Mapper.Context context) throws IOException, InterruptedException {
 try {
@@ -76,7 +82,7 @@ public class KylinMapper 
extends Mapper.Context 
context) throws IOException, InterruptedException {
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index 2b63ce0..cb2d7a7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -29,7 +30,9 @@ import org.slf4j.LoggerFactory;
  */
 public class KylinReducer extends 
Reducer {
 

kylin git commit: refine mapper and reducer log

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21 80018874c -> d3ecb0d9c


refine mapper and reducer log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d3ecb0d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d3ecb0d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d3ecb0d9

Branch: refs/heads/yang21
Commit: d3ecb0d9c381dbb035c7cada7d3c798e24fef1d1
Parents: 8001887
Author: Hongbin Ma 
Authored: Thu Dec 1 18:01:55 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 18:01:55 2016 +0800

--
 .../org/apache/kylin/engine/mr/KylinMapper.java | 17 +++---
 .../apache/kylin/engine/mr/KylinReducer.java| 17 +++---
 .../engine/mr/steps/BaseCuboidMapperBase.java   |  1 -
 .../kylin/engine/mr/steps/CuboidReducer.java| 21 ++--
 .../engine/mr/steps/HiveToBaseCuboidMapper.java | 10 ++
 .../engine/mr/steps/InMemCuboidMapper.java  | 34 
 .../engine/mr/steps/InMemCuboidReducer.java | 20 ++--
 .../kylin/engine/mr/steps/NDCuboidMapper.java   | 22 ++---
 8 files changed, 70 insertions(+), 72 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ecb0d9/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index a01f7a2..c5af2fe 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -18,18 +18,21 @@
 
 package org.apache.kylin.engine.mr;
 
-import java.io.IOException;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+
 /**
  */
 public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
+protected int mapCounter = 0;
+
 protected void bindCurrentConfiguration(Configuration conf) {
 logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 HadoopUtil.setCurrentConfiguration(conf);
@@ -38,6 +41,10 @@ public class KylinMapper 
extends Mapper.Context context) throws IOException, InterruptedException {
 try {
+if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 
0) {
+logger.info("Accepting Mapper Key with ordinal: " + 
mapCounter);
+}
+
 doMap(key, value, context);
 } catch (IOException ex) { // KYLIN-2170
 logger.error("", ex);
@@ -53,11 +60,11 @@ public class KylinMapper 
extends Mapper.Context context) throws IOException, InterruptedException {
 super.map(key, value, context);
 }
-
+
 @Override
 final protected void cleanup(Mapper.Context context) throws IOException, InterruptedException {
 try {
@@ -76,7 +83,7 @@ public class KylinMapper 
extends Mapper.Context 
context) throws IOException, InterruptedException {
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ecb0d9/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index 2b63ce0..83266ea 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -18,18 +18,22 @@
 
 package org.apache.kylin.engine.mr;
 
-import java.io.IOException;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+
 /**
  */
 public class 

[3/3] kylin git commit: KYLIN-2233 Support HBase 1.0.2

[mahongbin]

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11b7fff7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11b7fff7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11b7fff7

Branch: refs/heads/yang21-hbase102
Commit: 11b7fff75f3b8c4091741ee68f468d207a07fa4a
Parents: 215329e
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 16:18:42 2016 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 11 ++-
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 12 +++-
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/11b7fff7/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 1df14f4..ad70ee3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.8.1
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/11b7fff7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 0ade920..40da772 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/11b7fff7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of RegionScanner.nextRaw()
 // FIXME: will 

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/215329ea
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/215329ea
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/215329ea

Branch: refs/heads/yang21-hbase102
Commit: 215329ea084465e654bfc6e2e32e91f68a64254a
Parents: 8001887
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 16:15:27 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase102 5d7e7f239 -> 11b7fff75 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-  

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/215329ea
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/215329ea
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/215329ea

Branch: refs/heads/yang21-cdh5.7
Commit: 215329ea084465e654bfc6e2e32e91f68a64254a
Parents: 8001887
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 16:15:27 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-  

[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0291138
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0291138
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0291138

Branch: refs/heads/yang21-cdh5.7
Commit: c029113811fe3cd1e2d1539afb7ad5903987b3d2
Parents: 215329e
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Thu Dec 1 16:17:04 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0291138/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/c0291138/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/c0291138/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c0291138/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 0b6d6662e -> c02911381 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/215329ea
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/215329ea
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/215329ea

Branch: refs/heads/yang21-hbase1.x
Commit: 215329ea084465e654bfc6e2e32e91f68a64254a
Parents: 8001887
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 16:15:27 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 02f37bd82 -> 215329ea0 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/215329ea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-