date:20161205

[3/5] kylin git commit: KYLIN-1875 Rollback sample data back to star schema

2016-12-05 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/ee47f857/examples/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv
deleted file mode 100644
index aced0e3..000
--- a/examples/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv
+++ /dev/null
@@ -1,244 +0,0 @@
-AD,42.546245,1.601554,Andorra
-AE,23.424076,53.847818,United Arab Emirates
-AF,33.93911,67.709953,Afghanistan
-AG,17.060816,-61.796428,Antigua and Barbuda
-AI,18.220554,-63.068615,Anguilla
-AL,41.153332,20.168331,Albania
-AM,40.069099,45.038189,Armenia
-AN,12.226079,-69.060087,Netherlands Antilles
-AO,-11.202692,17.873887,Angola
-AQ,-75.250973,-0.071389,Antarctica
-AR,-38.416097,-63.616672,Argentina
-AS,-14.270972,-170.132217,American Samoa
-AT,47.516231,14.550072,Austria
-AU,-25.274398,133.775136,Australia
-AW,12.52111,-69.968338,Aruba
-AZ,40.143105,47.576927,Azerbaijan
-BA,43.915886,17.679076,Bosnia and Herzegovina
-BB,13.193887,-59.543198,Barbados
-BD,23.684994,90.356331,Bangladesh
-BE,50.503887,4.469936,Belgium
-BF,12.238333,-1.561593,Burkina Faso
-BG,42.733883,25.48583,Bulgaria
-BH,25.930414,50.637772,Bahrain
-BI,-3.373056,29.918886,Burundi
-BJ,9.30769,2.315834,Benin
-BM,32.321384,-64.75737,Bermuda
-BN,4.535277,114.727669,Brunei
-BO,-16.290154,-63.588653,Bolivia
-BR,-14.235004,-51.92528,Brazil
-BS,25.03428,-77.39628,Bahamas
-BT,27.514162,90.433601,Bhutan
-BV,-54.423199,3.413194,Bouvet Island
-BW,-22.328474,24.684866,Botswana
-BY,53.709807,27.953389,Belarus
-BZ,17.189877,-88.49765,Belize
-CA,56.130366,-106.346771,Canada
-CC,-12.164165,96.870956,Cocos [Keeling] Islands
-CD,-4.038333,21.758664,Congo [DRC]
-CF,6.61,20.939444,Central African Republic
-CG,-0.228021,15.827659,Congo [Republic]
-CH,46.818188,8.227512,Switzerland
-CI,7.539989,-5.54708,C?te d'Ivoire
-CK,-21.236736,-159.777671,Cook Islands
-CL,-35.675147,-71.542969,Chile
-CM,7.369722,12.354722,Cameroon
-CN,35.86166,104.195397,China
-CO,4.570868,-74.297333,Colombia
-CR,9.748917,-83.753428,Costa Rica
-CU,21.521757,-77.781167,Cuba
-CV,16.002082,-24.013197,Cape Verde
-CX,-10.447525,105.690449,Christmas Island
-CY,35.126413,33.429859,Cyprus
-CZ,49.817492,15.472962,Czech Republic
-DE,51.165691,10.451526,Germany
-DJ,11.825138,42.590275,Djibouti
-DK,56.26392,9.501785,Denmark
-DM,15.414999,-61.370976,Dominica
-DO,18.735693,-70.162651,Dominican Republic
-DZ,28.033886,1.659626,Algeria
-EC,-1.831239,-78.183406,Ecuador
-EE,58.595272,25.013607,Estonia
-EG,26.820553,30.802498,Egypt
-EH,24.215527,-12.885834,Western Sahara
-ER,15.179384,39.782334,Eritrea
-ES,40.463667,-3.74922,Spain
-ET,9.145,40.489673,Ethiopia
-FI,61.92411,25.748151,Finland
-FJ,-16.578193,179.414413,Fiji
-FK,-51.796253,-59.523613,Falkland Islands [Islas Malvinas]
-FM,7.425554,150.550812,Micronesia
-FO,61.892635,-6.911806,Faroe Islands
-FR,46.227638,2.213749,France
-GA,-0.803689,11.609444,Gabon
-GB,55.378051,-3.435973,United Kingdom
-GD,12.262776,-61.604171,Grenada
-GE,42.315407,43.356892,Georgia
-GF,3.933889,-53.125782,French Guiana
-GG,49.465691,-2.585278,Guernsey
-GH,7.946527,-1.023194,Ghana
-GI,36.137741,-5.345374,Gibraltar
-GL,71.706936,-42.604303,Greenland
-GM,13.443182,-15.310139,Gambia
-GN,9.945587,-9.696645,Guinea
-GP,16.995971,-62.067641,Guadeloupe
-GQ,1.650801,10.267895,Equatorial Guinea
-GR,39.074208,21.824312,Greece
-GS,-54.429579,-36.587909,South Georgia and the South Sandwich Islands
-GT,15.783471,-90.230759,Guatemala
-GU,13.444304,144.793731,Guam
-GW,11.803749,-15.180413,Guinea-Bissau
-GY,4.860416,-58.93018,Guyana
-GZ,31.354676,34.308825,Gaza Strip
-HK,22.396428,114.109497,Hong Kong
-HM,-53.08181,73.504158,Heard Island and McDonald Islands
-HN,15.19,-86.241905,Honduras
-HR,45.1,15.2,Croatia
-HT,18.971187,-72.285215,Haiti
-HU,47.162494,19.503304,Hungary
-ID,-0.789275,113.921327,Indonesia
-IE,53.41291,-8.24389,Ireland
-IL,31.046051,34.851612,Israel
-IM,54.236107,-4.548056,Isle of Man
-IN,20.593684,78.96288,India
-IO,-6.343194,71.876519,British Indian Ocean Territory
-IQ,33.223191,43.679291,Iraq
-IR,32.427908,53.688046,Iran
-IS,64.963051,-19.020835,Iceland
-IT,41.87194,12.56738,Italy
-JE,49.214439,-2.13125,Jersey
-JM,18.109581,-77.297508,Jamaica
-JO,30.585164,36.238414,Jordan
-JP,36.204824,138.252924,Japan
-KE,-0.023559,37.906193,Kenya
-KG,41.20438,74.766098,Kyrgyzstan
-KH,12.565679,104.990963,Cambodia
-KI,-3.370417,-168.734039,Kiribati
-KM,-11.875001,43.872219,Comoros
-KN,17.357822,-62.782998,Saint Kitts and Nevis
-KP,40.339852,127.510093,North Korea
-KR,35.907757,127.766922,South Korea
-KW,29.31166,47.481766,Kuwait
-KY,19.513469,-80.566956,Cayman Islands
-KZ,48.019573,66.923684,Kazakhstan
-LA,19.85627,102.495496,Laos
-LB,33.854721,35.862285,Lebanon
-LC,13.909444,-60.978893,Saint Lucia
-LI,47.166,9.555373,Liechtenstein
-LK,7.873054,80.771797,Sri Lanka
-LR,6.428055,-9.429499,Liberia
-LS,-29.609988,28.233608,Lesotho

[2/5] kylin git commit: KYLIN-1875 Rollback sample data back to star schema

2016-12-05 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/ee47f857/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index e13b7ad..dffa43b 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,14/12/2012,Others,88750,0,11,36.2828,4,1349,10002313,ANALYST,Beijing
-1,28/08/2012,Others,175750,0,13,23.8563,20,1927,10004376,ANALYST,Beijing
-2,16/02/2012,ABIN,148324,15,13,88.3418,18,1005,10006710,ADMIN,Shanghai
-3,19/10/2013,FP-non GTC,37831,0,13,47.3015,3,1209,10003717,ANALYST,Beijing
-4,22/10/2012,Others,140746,100,11,83.454,16,1154,10006076,ADMIN,Shanghai
-5,25/01/2013,FP-GTC,16509,0,-99,22.9896,11,1372,10007436,ADMIN,Shanghai
-6,04/04/2013,Others,963,0,13,88.5907,11,1648,10009869,MODELER,Hongkong
-7,11/04/2012,Others,15687,0,15,88.194,19,1866,1400,ADMIN,Shanghai
-8,19/01/2013,ABIN,60606,3,13,77.9727,13,1936,1687,MODELER,Hongkong
-9,30/04/2012,FP-non 
GTC,106246,0,14,52.295,17,1047,10009223,MODELER,Hongkong
-10,03/02/2013,Auction,45333,0,16,56.3584,10,1470,10006759,MODELER,Hongkong
-11,02/09/2012,FP-GTC,25147,0,-99,15.3553,8,1203,1062,ANALYST,Beijing
-12,30/07/2013,Others,24760,0,16,25.077,5,1382,10003539,ADMIN,Shanghai
-13,17/04/2012,Auction,31519,0,16,72.7384,11,1069,10001245,ADMIN,Shanghai
-14,30/04/2013,FP-GTC,51582,0,15,75.82,6,1085,10008148,ANALYST,Beijing
-15,23/03/2013,FP-GTC,16509,0,15,8.6653,15,1559,10008079,MODELER,Hongkong
-16,01/04/2013,ABIN,175750,3,16,64.2802,16,1507,10007797,MODELER,Hongkong
-17,12/07/2013,ABIN,50677,0,-99,24.5987,17,1938,10006224,ADMIN,Shanghai
-18,23/04/2012,FP-GTC,45333,0,5,72.6553,15,1863,10003824,MODELER,Hongkong
-19,19/12/2012,ABIN,3838,0,12,87.4142,4,1612,10007157,ADMIN,Shanghai
-20,15/08/2012,FP-GTC,62179,0,16,67.4238,15,1839,10003452,MODELER,Hongkong
-21,22/08/2013,Auction,1357,0,5,18.36,9,1303,1114,ANALYST,Beijing
-22,23/08/2013,ABIN,31519,0,5,67.6344,2,1877,10007417,ANALYST,Beijing
-23,12/09/2013,FP-GTC,223,0,14,71.7898,15,1848,1627,MODELER,Hongkong
-24,14/03/2012,FP-GTC,132939,0,15,26.1496,20,1815,10005292,MODELER,Hongkong
-25,13/07/2013,Others,67703,3,16,83.9661,2,1858,10008123,MODELER,Hongkong
-26,15/12/2013,FP-non GTC,100847,0,5,34.3451,16,1387,10007806,ADMIN,Shanghai
-27,30/08/2012,Auction,62179,0,11,21.0496,9,1393,10007360,ANALYST,Beijing
-28,10/01/2013,FP-GTC,48027,0,13,23.4142,20,1757,10002135,MODELER,Hongkong
-29,25/01/2013,ABIN,145970,0,5,26.7842,6,1402,10009682,ANALYST,Beijing
-30,05/07/2012,Auction,48027,0,-99,46.732,5,1972,10009522,MODELER,Hongkong
-31,07/11/2012,FP-GTC,20213,0,-99,19.727,8,1873,10009944,ANALYST,Beijing
-32,20/06/2013,ABIN,100847,0,5,76.2783,1,1409,10005395,ADMIN,Shanghai
-33,08/10/2012,ABIN,26262,0,5,48.7447,4,1029,10001836,ADMIN,Shanghai
-34,30/04/2013,ABIN,87118,0,15,63.2187,11,1529,1928,ANALYST,Beijing
-35,24/04/2013,FP-non 
GTC,20213,0,16,39.5057,19,1329,1768,ANALYST,Beijing
-36,13/05/2012,Others,164261,0,13,84.1246,15,1688,1727,ADMIN,Shanghai
-37,02/08/2013,ABIN,31519,0,5,37.1504,14,1905,1719,MODELER,Hongkong
-38,03/04/2012,Auction,164261,0,11,92.0974,13,1341,10001337,ADMIN,Shanghai
-39,02/09/2012,FP-GTC,31519,0,14,87.7829,20,1460,10008484,ANALYST,Beijing
-40,05/10/2013,ABIN,13987,0,16,74.0719,4,1381,10002975,MODELER,Hongkong
-41,13/01/2012,FP-GTC,48027,0,15,71.2637,14,1915,10005778,ADMIN,Shanghai
-42,01/03/2013,Others,13836,0,14,16.7288,14,1953,10009385,ADMIN,Shanghai
-43,12/09/2012,FP-non 
GTC,16509,0,5,12.2933,20,1277,10002286,MODELER,Hongkong
-44,07/02/2012,Auction,45333,0,5,64.977,16,1040,10005583,MODELER,Hongkong
-45,14/12/2013,FP-non 
GTC,158798,0,16,72.4413,18,1500,10007635,MODELER,Hongkong
-46,13/10/2013,Auction,31519,0,5,79.3053,17,1816,10006423,MODELER,Hongkong
-47,11/06/2012,ABIN,4943,0,5,11.6942,15,1696,10001237,ANALYST,Beijing
-48,18/10/2012,ABIN,80053,0,-99,54.0933,4,1604,10006851,MODELER,Hongkong
-49,03/11/2012,Auction,20886,0,5,9.8258,8,1764,10001220,MODELER,Hongkong
-50,15/01/2012,Auction,44079,0,14,13.0371,19,1343,10001259,MODELER,Hongkong
-51,03/07/2012,FP-GTC,159184,0,-99,92.5314,4,1724,10009008,ADMIN,Shanghai
-52,16/02/2012,FP-GTC,314,0,13,49.1825,20,1334,10004005,ADMIN,Shanghai
-53,17/06/2012,FP-non GTC,31387,3,12,0.6677,5,1603,10002595,ANALYST,Beijing
-54,15/07/2012,FP-non 
GTC,32876,0,-99,50.9634,11,1248,10006890,ANALYST,Beijing
-55,24/07/2012,FP-GTC,150265,15,16,57.5645,6,1653,10007269,ADMIN,Shanghai
-56,06/10/2012,Auction,145970,0,5,44.3091,16,1082,10001640,ADMIN,Shanghai
-57,14/06/2013,ABIN,80053,0,5,20.2388,19,1632,10002068,MODELER,Hongkong
-58,10/10/2013,FP-non

[5/5] kylin git commit: KYLIN-1875 Rollback sample data back to star schema

2016-12-05 Thread liyang

KYLIN-1875 Rollback sample data back to star schema


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ee47f857
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ee47f857
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ee47f857

Branch: refs/heads/yang22
Commit: ee47f8572941735c443d034b548e4494f70f749e
Parents: 7699973
Author: Li Yang 
Authored: Tue Dec 6 15:09:51 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 6 15:09:51 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql   |77 +-
 .../sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv  | 1 -
 .../sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv  |   244 -
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 .../template/cube_desc/kylin_sales_cube.json|   157 +-
 .../template/model_desc/kylin_sales_model.json  |50 +-
 .../template/table/DEFAULT.KYLIN_ACCOUNT.json   |28 -
 .../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json | 4 +-
 .../template/table/DEFAULT.KYLIN_COUNTRY.json   |24 -
 .../template/table/DEFAULT.KYLIN_SALES.json |22 +-
 10 files changed, 10081 insertions(+), 20525 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ee47f857/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index bdfc4d7..bc78802 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -20,11 +20,11 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_CAL_DT;
 
 CREATE TABLE DEFAULT.KYLIN_CAL_DT
 (
-CAL_DT date COMMENT 'Date, PK'
-,YEAR_BEG_DT date COMMENT 'YEAR Begin Date'
-,QTR_BEG_DT date COMMENT 'Quarter Begin Date'
-,MONTH_BEG_DT date COMMENT 'Month Begin Date'
-,WEEK_BEG_DT date COMMENT 'Week Begin Date'
+CAL_DT date
+,YEAR_BEG_DT date
+,QTR_BEG_DT date
+,MONTH_BEG_DT date
+,WEEK_BEG_DT date
 ,AGE_FOR_YEAR_ID smallint
 ,AGE_FOR_QTR_ID smallint
 ,AGE_FOR_MONTH_ID smallint
@@ -121,7 +121,6 @@ CAL_DT date COMMENT 'Date, PK'
 ,KYLIN_CAL_DT_UPD_DATE string
 ,KYLIN_CAL_DT_UPD_USER string
 )
-COMMENT 'Date Dimension Table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
@@ -129,28 +128,28 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_CATEGORY_GROUPINGS;
 
 CREATE TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS
 (
-LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
+LEAF_CATEG_ID bigint
 ,LEAF_CATEG_NAME string
-,SITE_ID int COMMENT 'Site ID, PK'
+,SITE_ID int
 ,CATEG_BUSN_MGR string
 ,CATEG_BUSN_UNIT string
 ,REGN_CATEG string
-,USER_DEFINED_FIELD1 string COMMENT 'User Defined Field1'
-,USER_DEFINED_FIELD3 string COMMENT 'User Defined Field3'
+,USER_DEFINED_FIELD1 string
+,USER_DEFINED_FIELD3 string
 ,KYLIN_GROUPINGS_CRE_DATE string
-,KYLIN_GROUPINGS_UPD_DATE string COMMENT 'Last Updated Date'
+,KYLIN_GROUPINGS_UPD_DATE string
 ,KYLIN_GROUPINGS_CRE_USER string
-,KYLIN_GROUPINGS_UPD_USER string COMMENT 'Last Updated User'
+,KYLIN_GROUPINGS_UPD_USER string
 ,META_CATEG_ID decimal
-,META_CATEG_NAME string COMMENT 'Level1 Category'
+,META_CATEG_NAME string
 ,CATEG_LVL2_ID decimal
 ,CATEG_LVL3_ID decimal
 ,CATEG_LVL4_ID decimal
 ,CATEG_LVL5_ID decimal
 ,CATEG_LVL6_ID decimal
 ,CATEG_LVL7_ID decimal
-,CATEG_LVL2_NAME string COMMENT 'Level2 Category'
-,CATEG_LVL3_NAME string COMMENT 'Level3 Category'
+,CATEG_LVL2_NAME string
+,CATEG_LVL3_NAME string
 ,CATEG_LVL4_NAME string
 ,CATEG_LVL5_NAME string
 ,CATEG_LVL6_NAME string
@@ -166,32 +165,6 @@ LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
 ,SRC_ID tinyint
 ,BSNS_VRTCL_NAME string
 )
-COMMENT 'Detail category inforamtion, Dimension Table'
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS TEXTFILE;
-
-DROP TABLE IF EXISTS DEFAULT.KYLIN_COUNTRY;
-
-CREATE TABLE DEFAULT.KYLIN_COUNTRY
-(
-COUNTRY string
-,LATITUDE double
-,LONGITUDE double
-,NAME string
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS TEXTFILE;
-
-DROP TABLE IF EXISTS DEFAULT.KYLIN_ACCOUNT;
-
-CREATE TABLE DEFAULT.KYLIN_ACCOUNT
-(
-ACCOUNT_ID bigint
-,ACCOUNT_BUYER_LEVEL int COMMENT 'Account Buyer Level'
-,ACCOUNT_SELLER_LEVEL int COMMENT 'Account Seller Level'
-,ACCOUNT_COUNTRY string COMMENT 'Account Country'
-,ACCOUNT_CONTACT string COMMENT 'Account Contact Info'
-)
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
@@ -200,24 +173,20 @@ DROP TABLE IF EXISTS DEFAULT.KYLIN_SALES;
 CREATE TABLE DEFAULT.KYLIN_SALES
 (
 TRANS_ID bigint
-,PART_DT date COMMENT 'Order Date'
-,LSTG_FORMAT_NAME string COMMENT 'Order Transaction Type'
-,LEAF_CATEG_ID bigint COMMENT 'Category ID'
-,LSTG_SITE_ID int COMMENT 'Site ID'
+,PART_DT date
+,LSTG_FORMAT_NAME string
+,LEAF_CATEG_ID bigint
+,LSTG_SITE_ID

[4/5] kylin git commit: KYLIN-1875 Rollback sample data back to star schema

2016-12-05 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/ee47f857/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
deleted file mode 100644
index 253afbf..000
--- a/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
+++ /dev/null
@@ -1,1 +0,0 @@
-1000,1,4,FR,N/A
-1001,0,1,DE,N/A
-1002,0,1,IT,N/A
-1003,4,2,DE,N/A
-1004,2,3,JP,N/A
-1005,0,1,CN,N/A
-1006,2,3,JP,N/A
-1007,0,2,GB,N/A
-1008,5,3,US,N/A
-1009,1,3,US,N/A
-1010,1,1,IT,N/A
-1011,2,4,FR,N/A
-1012,3,1,CN,N/A
-1013,1,1,FR,N/A
-1014,5,3,CN,N/A
-1015,3,5,IT,N/A
-1016,4,5,JP,N/A
-1017,3,4,US,N/A
-1018,0,5,CN,N/A
-1019,4,4,GB,N/A
-1020,4,4,IT,N/A
-1021,1,2,FR,N/A
-1022,2,2,DE,N/A
-1023,0,1,GB,N/A
-1024,3,4,IT,N/A
-1025,4,4,JP,N/A
-1026,5,4,IT,N/A
-1027,2,4,FR,N/A
-1028,2,2,GB,N/A
-1029,1,1,GB,N/A
-1030,5,5,CN,N/A
-1031,4,2,CN,N/A
-1032,1,3,CN,N/A
-1033,2,1,JP,N/A
-1034,2,1,JP,N/A
-1035,4,2,US,N/A
-1036,5,4,DE,N/A
-1037,2,4,DE,N/A
-1038,1,3,US,N/A
-1039,3,5,US,N/A
-1040,2,3,GB,N/A
-1041,1,5,IT,N/A
-1042,5,5,CN,N/A
-1043,5,1,IT,N/A
-1044,3,4,IT,N/A
-1045,1,3,US,N/A
-1046,3,3,JP,N/A
-1047,2,2,FR,N/A
-1048,5,4,JP,N/A
-1049,3,1,JP,N/A
-1050,3,5,JP,N/A
-1051,5,1,US,N/A
-1052,3,5,DE,N/A
-1053,1,3,JP,N/A
-1054,5,1,GB,N/A
-1055,2,1,GB,N/A
-1056,2,3,CN,N/A
-1057,1,5,US,N/A
-1058,2,2,CN,N/A
-1059,2,4,FR,N/A
-1060,0,3,FR,N/A
-1061,0,5,FR,N/A
-1062,1,1,US,N/A
-1063,4,5,DE,N/A
-1064,2,5,GB,N/A
-1065,0,3,GB,N/A
-1066,1,1,IT,N/A
-1067,0,2,GB,N/A
-1068,4,1,IT,N/A
-1069,0,4,FR,N/A
-1070,3,4,CN,N/A
-1071,3,5,GB,N/A
-1072,4,3,CN,N/A
-1073,0,4,IT,N/A
-1074,5,3,GB,N/A
-1075,0,4,JP,N/A
-1076,2,5,JP,N/A
-1077,0,3,CN,N/A
-1078,4,3,GB,N/A
-1079,4,5,US,N/A
-1080,4,2,JP,N/A
-1081,5,2,CN,N/A
-1082,4,1,JP,N/A
-1083,0,4,GB,N/A
-1084,1,4,DE,N/A
-1085,4,4,FR,N/A
-1086,2,5,JP,N/A
-1087,0,1,IT,N/A
-1088,0,2,FR,N/A
-1089,3,2,DE,N/A
-1090,5,2,IT,N/A
-1091,0,3,GB,N/A
-1092,1,4,DE,N/A
-1093,2,1,IT,N/A
-1094,4,3,JP,N/A
-1095,1,1,DE,N/A
-1096,0,5,JP,N/A
-1097,3,3,CN,N/A
-1098,5,3,JP,N/A
-1099,3,1,GB,N/A
-1100,5,1,CN,N/A
-1101,2,3,JP,N/A
-1102,5,1,IT,N/A
-1103,5,2,GB,N/A
-1104,3,5,GB,N/A
-1105,3,1,DE,N/A
-1106,3,5,GB,N/A
-1107,5,3,DE,N/A
-1108,4,1,CN,N/A
-1109,4,3,GB,N/A
-1110,4,1,FR,N/A
-1111,3,5,FR,N/A
-1112,1,1,GB,N/A
-1113,5,4,FR,N/A
-1114,2,2,DE,N/A
-1115,4,3,GB,N/A
-1116,0,3,FR,N/A
-1117,1,4,CN,N/A
-1118,4,4,IT,N/A
-1119,0,2,CN,N/A
-1120,5,2,DE,N/A
-1121,0,2,GB,N/A
-1122,4,3,CN,N/A
-1123,1,4,FR,N/A
-1124,0,2,CN,N/A
-1125,4,5,US,N/A
-1126,0,4,GB,N/A
-1127,4,3,CN,N/A
-1128,4,3,GB,N/A
-1129,1,4,JP,N/A
-1130,2,2,JP,N/A
-1131,0,3,IT,N/A
-1132,4,1,GB,N/A
-1133,1,5,US,N/A
-1134,3,3,DE,N/A
-1135,3,2,IT,N/A
-1136,2,2,JP,N/A
-1137,0,3,CN,N/A
-1138,3,1,FR,N/A
-1139,2,3,FR,N/A
-1140,3,1,IT,N/A
-1141,0,3,US,N/A
-1142,3,4,FR,N/A
-1143,4,5,DE,N/A
-1144,3,3,JP,N/A
-1145,5,5,DE,N/A
-1146,2,5,FR,N/A
-1147,0,5,IT,N/A
-1148,2,5,US,N/A
-1149,0,3,CN,N/A
-1150,2,2,GB,N/A
-1151,5,5,CN,N/A
-1152,2,2,IT,N/A
-1153,3,2,FR,N/A
-1154,5,4,FR,N/A
-1155,3,5,FR,N/A
-1156,5,2,GB,N/A
-1157,1,2,IT,N/A
-1158,3,3,JP,N/A
-1159,5,1,IT,N/A
-1160,3,4,CN,N/A
-1161,2,5,IT,N/A
-1162,4,5,CN,N/A
-1163,3,5,FR,N/A
-1164,1,2,FR,N/A
-1165,0,3,GB,N/A
-1166,5,4,CN,N/A
-1167,4,5,IT,N/A
-1168,3,2,IT,N/A
-1169,3,2,DE,N/A
-1170,2,2,DE,N/A
-1171,3,1,IT,N/A
-1172,3,1,IT,N/A
-1173,3,5,CN,N/A
-1174,5,2,US,N/A
-1175,2,3,FR,N/A
-1176,5,5,JP,N/A
-1177,1,4,DE,N/A
-1178,4,5,GB,N/A
-1179,5,3,FR,N/A
-1180,2,3,GB,N/A
-1181,0,3,IT,N/A
-1182,3,2,JP,N/A
-1183,5,5,IT,N/A
-1184,4,1,IT,N/A
-1185,3,2,IT,N/A
-1186,3,2,CN,N/A
-1187,2,5,IT,N/A
-1188,3,2,JP,N/A
-1189,4,4,IT,N/A
-1190,4,3,GB,N/A
-1191,3,4,JP,N/A
-1192,2,2,FR,N/A
-1193,5,1,FR,N/A
-1194,5,2,US,N/A
-1195,2,5,FR,N/A
-1196,1,2,FR,N/A
-1197,5,3,FR,N/A
-1198,1,4,GB,N/A
-1199,1,3,FR,N/A
-1200,4,5,IT,N/A
-1201,3,1,GB,N/A
-1202,0,3,GB,N/A
-1203,4,3,DE,N/A
-1204,4,2,GB,N/A
-1205,4,2,DE,N/A
-1206,5,2,GB,N/A
-1207,1,5,DE,N/A
-1208,4,5,GB,N/A
-1209,0,3,FR,N/A
-1210,1,5,FR,N/A
-1211,3,2,DE,N/A
-1212,5,1,JP,N/A
-1213,0,4,CN,N/A
-1214,3,1,JP,N/A
-1215,5,1,CN,N/A
-1216,4,2,FR,N/A

[kylin] Git Push Summary

2016-12-05 Thread mahongbin

Repository: kylin
Updated Tags:  refs/tags/yinlian-1104 [deleted] f03f26af9

[kylin] Git Push Summary

2016-12-05 Thread mahongbin

Repository: kylin
Updated Tags:  refs/tags/SAIC-1104 [deleted] f03f26af9

kylin git commit: refine sample cube, add kind=LOOKUP attr

2016-12-05 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1875 706d6137f -> 3e12fbb74


refine sample cube, add kind=LOOKUP attr


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3e12fbb7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3e12fbb7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3e12fbb7

Branch: refs/heads/KYLIN-1875
Commit: 3e12fbb744928704f128ace71ac5d480817946c1
Parents: 706d613
Author: Li Yang 
Authored: Tue Dec 6 14:39:43 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 6 14:39:43 2016 +0800

--
 examples/sample_cube/template/model_desc/kylin_sales_model.json | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3e12fbb7/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 2d3cfb6..cce360f 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -18,6 +18,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_ACCOUNT",
 "alias" : "BUYER_ACCOUNT",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "ACCOUNT_ID" ],
@@ -26,6 +27,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_ACCOUNT",
 "alias" : "SELLER_ACCOUNT",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "ACCOUNT_ID" ],
@@ -34,6 +36,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_COUNTRY",
 "alias" : "BUYER_COUNTRY",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "COUNTRY" ],
@@ -42,6 +45,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_COUNTRY",
 "alias" : "SELLER_COUNTRY",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "COUNTRY" ],

svn commit: r1772835 - in /kylin/site: community/index.html docs/howto/howto_use_restapi.html docs15/howto/howto_use_restapi.html docs16/howto/howto_use_restapi.html feed.xml

2016-12-05 Thread lidong

Author: lidong
Date: Tue Dec  6 06:03:06 2016
New Revision: 1772835

URL: http://svn.apache.org/viewvc?rev=1772835=rev
Log:
update committer info

Modified:
kylin/site/community/index.html
kylin/site/docs/howto/howto_use_restapi.html
kylin/site/docs15/howto/howto_use_restapi.html
kylin/site/docs16/howto/howto_use_restapi.html
kylin/site/feed.xml

Modified: kylin/site/community/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/community/index.html?rev=1772835=1772834=1772835=diff
==
--- kylin/site/community/index.html (original)
+++ kylin/site/community/index.html Tue Dec  6 06:03:06 2016
@@ -374,6 +374,12 @@
   http://github.com/kyotoYaho;>http://github.com/kyotoYaho.png; />
   committer
 
+
+  Billy Liu (åä¸é¸£)
+  http://home.apache.org/phonebook.html?uid=billyliu;>billyliu
+  http://github.com/yiming187;>http://github.com/yiming187.png; />
+  committer
+
   
 
 

Modified: kylin/site/docs/howto/howto_use_restapi.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs/howto/howto_use_restapi.html?rev=1772835=1772834=1772835=diff
==
--- kylin/site/docs/howto/howto_use_restapi.html (original)
+++ kylin/site/docs/howto/howto_use_restapi.html Tue Dec  6 06:03:06 2016
@@ -2877,7 +2877,7 @@ Get descriptor for specified cube instan
 (Same as âResume Jobâ)
 
 Get job step output
-GET 
/{jobId}/steps/{stepId}/output
+GET 
/jobs/{jobId}/steps/{stepId}/output
 
 Path Variable
 

Modified: kylin/site/docs15/howto/howto_use_restapi.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs15/howto/howto_use_restapi.html?rev=1772835=1772834=1772835=diff
==
--- kylin/site/docs15/howto/howto_use_restapi.html (original)
+++ kylin/site/docs15/howto/howto_use_restapi.html Tue Dec  6 06:03:06 2016
@@ -3015,7 +3015,7 @@ Get descriptor for specified cube instan
 (Same as âResume Jobâ)
 
 Get job step output
-GET 
/kylin/api/{jobId}/steps/{stepId}/output
+GET 
/kylin/api/jobs/{jobId}/steps/{stepId}/output
 
 Path Variable
 

Modified: kylin/site/docs16/howto/howto_use_restapi.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs16/howto/howto_use_restapi.html?rev=1772835=1772834=1772835=diff
==
--- kylin/site/docs16/howto/howto_use_restapi.html (original)
+++ kylin/site/docs16/howto/howto_use_restapi.html Tue Dec  6 06:03:06 2016
@@ -2972,7 +2972,7 @@ Get descriptor for specified cube instan
 (Same as âResume Jobâ)
 
 Get job step output
-GET 
/kylin/api/{jobId}/steps/{stepId}/output
+GET 
/kylin/api/jobs/{jobId}/steps/{stepId}/output
 
 Path Variable
 

Modified: kylin/site/feed.xml
URL: 
http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1772835=1772834=1772835=diff
==
--- kylin/site/feed.xml (original)
+++ kylin/site/feed.xml Tue Dec  6 06:03:06 2016
@@ -19,8 +19,8 @@
 Apache Kylin Home
 http://kylin.apache.org/
 http://kylin.apache.org/feed.xml; rel="self" 
type="application/rss+xml"/>
-Sat, 03 Dec 2016 13:12:45 -0800
-Sat, 03 Dec 2016 13:12:45 -0800
+Tue, 06 Dec 2016 05:59:08 -0800
+Tue, 06 Dec 2016 05:59:08 -0800
 Jekyll v2.5.3

[kylin] Git Push Summary

2016-12-05 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang22 [created] 76999735f

[kylin] Git Push Summary

2016-12-05 Thread mahongbin

Repository: kylin
Updated Tags:  refs/tags/yang22 [deleted] 76999735f

[2/3] kylin git commit: minor, generate item_count value for sample data

2016-12-05 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/d2aaf270/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index 7349b37..e13b7ad 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,1349,10002313,ANALYST,Beijing
-1,2012-08-28,Others,175750,0,13,23.8563,0,1927,10004376,ANALYST,Beijing
-2,2012-02-16,ABIN,148324,15,13,88.3418,0,1005,10006710,ADMIN,Shanghai
-3,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,1209,10003717,ANALYST,Beijing
-4,2012-10-22,Others,140746,100,11,83.454,0,1154,10006076,ADMIN,Shanghai
-5,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,1372,10007436,ADMIN,Shanghai
-6,2013-04-04,Others,963,0,13,88.5907,0,1648,10009869,MODELER,Hongkong
-7,2012-04-11,Others,15687,0,15,88.194,0,1866,1400,ADMIN,Shanghai
-8,2013-01-19,ABIN,60606,3,13,77.9727,0,1936,1687,MODELER,Hongkong
-9,2012-04-30,FP-non GTC,106246,0,14,52.295,0,1047,10009223,MODELER,Hongkong
-10,2013-02-03,Auction,45333,0,16,56.3584,0,1470,10006759,MODELER,Hongkong
-11,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,1203,1062,ANALYST,Beijing
-12,2013-07-30,Others,24760,0,16,25.077,0,1382,10003539,ADMIN,Shanghai
-13,2012-04-17,Auction,31519,0,16,72.7384,0,1069,10001245,ADMIN,Shanghai
-14,2013-04-30,FP-GTC,51582,0,15,75.82,0,1085,10008148,ANALYST,Beijing
-15,2013-03-23,FP-GTC,16509,0,15,8.6653,0,1559,10008079,MODELER,Hongkong
-16,2013-04-01,ABIN,175750,3,16,64.2802,0,1507,10007797,MODELER,Hongkong
-17,2013-07-12,ABIN,50677,0,-99,24.5987,0,1938,10006224,ADMIN,Shanghai
-18,2012-04-23,FP-GTC,45333,0,5,72.6553,0,1863,10003824,MODELER,Hongkong
-19,2012-12-19,ABIN,3838,0,12,87.4142,0,1612,10007157,ADMIN,Shanghai
-20,2012-08-15,FP-GTC,62179,0,16,67.4238,0,1839,10003452,MODELER,Hongkong
-21,2013-08-22,Auction,1357,0,5,18.36,0,1303,1114,ANALYST,Beijing
-22,2013-08-23,ABIN,31519,0,5,67.6344,0,1877,10007417,ANALYST,Beijing
-23,2013-09-12,FP-GTC,223,0,14,71.7898,0,1848,1627,MODELER,Hongkong
-24,2012-03-14,FP-GTC,132939,0,15,26.1496,0,1815,10005292,MODELER,Hongkong
-25,2013-07-13,Others,67703,3,16,83.9661,0,1858,10008123,MODELER,Hongkong
-26,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,1387,10007806,ADMIN,Shanghai
-27,2012-08-30,Auction,62179,0,11,21.0496,0,1393,10007360,ANALYST,Beijing
-28,2013-01-10,FP-GTC,48027,0,13,23.4142,0,1757,10002135,MODELER,Hongkong
-29,2013-01-25,ABIN,145970,0,5,26.7842,0,1402,10009682,ANALYST,Beijing
-30,2012-07-05,Auction,48027,0,-99,46.732,0,1972,10009522,MODELER,Hongkong
-31,2012-11-07,FP-GTC,20213,0,-99,19.727,0,1873,10009944,ANALYST,Beijing
-32,2013-06-20,ABIN,100847,0,5,76.2783,0,1409,10005395,ADMIN,Shanghai
-33,2012-10-08,ABIN,26262,0,5,48.7447,0,1029,10001836,ADMIN,Shanghai
-34,2013-04-30,ABIN,87118,0,15,63.2187,0,1529,1928,ANALYST,Beijing
-35,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,1329,1768,ANALYST,Beijing
-36,2012-05-13,Others,164261,0,13,84.1246,0,1688,1727,ADMIN,Shanghai
-37,2013-08-02,ABIN,31519,0,5,37.1504,0,1905,1719,MODELER,Hongkong
-38,2012-04-03,Auction,164261,0,11,92.0974,0,1341,10001337,ADMIN,Shanghai
-39,2012-09-02,FP-GTC,31519,0,14,87.7829,0,1460,10008484,ANALYST,Beijing
-40,2013-10-05,ABIN,13987,0,16,74.0719,0,1381,10002975,MODELER,Hongkong
-41,2012-01-13,FP-GTC,48027,0,15,71.2637,0,1915,10005778,ADMIN,Shanghai
-42,2013-03-01,Others,13836,0,14,16.7288,0,1953,10009385,ADMIN,Shanghai
-43,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,1277,10002286,MODELER,Hongkong
-44,2012-02-07,Auction,45333,0,5,64.977,0,1040,10005583,MODELER,Hongkong
-45,2013-12-14,FP-non 
GTC,158798,0,16,72.4413,0,1500,10007635,MODELER,Hongkong
-46,2013-10-13,Auction,31519,0,5,79.3053,0,1816,10006423,MODELER,Hongkong
-47,2012-06-11,ABIN,4943,0,5,11.6942,0,1696,10001237,ANALYST,Beijing
-48,2012-10-18,ABIN,80053,0,-99,54.0933,0,1604,10006851,MODELER,Hongkong
-49,2012-11-03,Auction,20886,0,5,9.8258,0,1764,10001220,MODELER,Hongkong
-50,2012-01-15,Auction,44079,0,14,13.0371,0,1343,10001259,MODELER,Hongkong
-51,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,1724,10009008,ADMIN,Shanghai
-52,2012-02-16,FP-GTC,314,0,13,49.1825,0,1334,10004005,ADMIN,Shanghai
-53,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,1603,10002595,ANALYST,Beijing
-54,2012-07-15,FP-non 
GTC,32876,0,-99,50.9634,0,1248,10006890,ANALYST,Beijing
-55,2012-07-24,FP-GTC,150265,15,16,57.5645,0,1653,10007269,ADMIN,Shanghai
-56,2012-10-06,Auction,145970,0,5,44.3091,0,1082,10001640,ADMIN,Shanghai
-57,2013-06-14,ABIN,80053,0,5,20.2388,0,1632,10002068,MODELER,Hongkong
-58,2013-10-10,FP-non

[3/3] kylin git commit: minor, generate item_count value for sample data

2016-12-05 Thread mahongbin

minor, generate item_count value for sample data

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d2aaf270
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d2aaf270
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d2aaf270

Branch: refs/heads/master
Commit: d2aaf2705a9035be9dbef8205fc9a1358b4316c2
Parents: 9968c22
Author: Yiming Liu 
Authored: Fri Dec 2 16:39:00 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 6 13:50:28 2016 +0800

--
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 1 file changed, 1 insertions(+), 1 deletions(-)
--

kylin git commit: update committer info

2016-12-05 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/document 439379a17 -> 1dcf042af


update committer info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1dcf042a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1dcf042a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1dcf042a

Branch: refs/heads/document
Commit: 1dcf042af38c9a933e126bceb25a11a6479fb999
Parents: 439379a
Author: Billy Liu 
Authored: Tue Dec 6 12:05:44 2016 +0800
Committer: Billy Liu 
Committed: Tue Dec 6 12:05:44 2016 +0800

--
 website/_data/contributors.yml | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1dcf042a/website/_data/contributors.yml
--
diff --git a/website/_data/contributors.yml b/website/_data/contributors.yml
index e56c02f..f44194e 100644
--- a/website/_data/contributors.yml
+++ b/website/_data/contributors.yml
@@ -125,4 +125,8 @@
   githubId: kyotoYaho
   org: eBay
   role: committer
-
+- name: Billy Liu (åä¸é¸£)
+  apacheId: billyliu
+  githubId: yiming187
+  org: Kyligence Inc.
+  role: committer

kylin git commit: Thanks chitre.a...@gmail.com: typo in rest api doc

2016-12-05 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document 3c6a9a63d -> 439379a17


Thanks chitre.a...@gmail.com: typo in rest api doc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/439379a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/439379a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/439379a1

Branch: refs/heads/document
Commit: 439379a17c3c206f231cf67fcafe1b38f4284003
Parents: 3c6a9a6
Author: shaofengshi 
Authored: Tue Dec 6 11:50:23 2016 +0800
Committer: shaofengshi 
Committed: Tue Dec 6 11:50:23 2016 +0800

--
 website/_docs/howto/howto_use_restapi.md   | 2 +-
 website/_docs15/howto/howto_use_restapi.md | 2 +-
 website/_docs16/howto/howto_use_restapi.md | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/439379a1/website/_docs/howto/howto_use_restapi.md
--
diff --git a/website/_docs/howto/howto_use_restapi.md 
b/website/_docs/howto/howto_use_restapi.md
index 5924a2b..df5dcd8 100644
--- a/website/_docs/howto/howto_use_restapi.md
+++ b/website/_docs/howto/howto_use_restapi.md
@@ -858,7 +858,7 @@ Get descriptor for specified cube instance.
 (Same as "Resume Job")
 
 ## Get job step output
-`GET /{jobId}/steps/{stepId}/output`
+`GET /jobs/{jobId}/steps/{stepId}/output`
 
  Path Variable
 * jobId - `required` `string` Job id.

http://git-wip-us.apache.org/repos/asf/kylin/blob/439379a1/website/_docs15/howto/howto_use_restapi.md
--
diff --git a/website/_docs15/howto/howto_use_restapi.md 
b/website/_docs15/howto/howto_use_restapi.md
index 622a8e1..23e4116 100644
--- a/website/_docs15/howto/howto_use_restapi.md
+++ b/website/_docs15/howto/howto_use_restapi.md
@@ -873,7 +873,7 @@ Get descriptor for specified cube instance.
 (Same as "Resume Job")
 
 ## Get job step output
-`GET /kylin/api/{jobId}/steps/{stepId}/output`
+`GET /kylin/api/jobs/{jobId}/steps/{stepId}/output`
 
  Path Variable
 * jobId - `required` `string` Job id.

http://git-wip-us.apache.org/repos/asf/kylin/blob/439379a1/website/_docs16/howto/howto_use_restapi.md
--
diff --git a/website/_docs16/howto/howto_use_restapi.md 
b/website/_docs16/howto/howto_use_restapi.md
index ac25f85..604e905 100644
--- a/website/_docs16/howto/howto_use_restapi.md
+++ b/website/_docs16/howto/howto_use_restapi.md
@@ -881,7 +881,7 @@ Get descriptor for specified cube instance.
 (Same as "Resume Job")
 
 ## Get job step output
-`GET /kylin/api/{jobId}/steps/{stepId}/output`
+`GET /kylin/api/jobs/{jobId}/steps/{stepId}/output`
 
  Path Variable
 * jobId - `required` `string` Job id.

kylin git commit: KYLIN-2245 further minor refactor

2016-12-05 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master fc1e11aa8 -> 9968c2257


KYLIN-2245 further minor refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9968c225
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9968c225
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9968c225

Branch: refs/heads/master
Commit: 9968c2257c2a8605637a93830d1d95db94185dce
Parents: fc1e11a
Author: Li Yang 
Authored: Tue Dec 6 11:10:26 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 6 11:10:26 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |  2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |  3 ++-
 .../apache/kylin/metadata/model/ISegment.java   |  3 +++
 .../apache/kylin/metadata/model/Segments.java   | 15 +++
 .../test_case_data/sandbox/kylin_hive_conf.xml  | 20 +---
 5 files changed, 10 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index a3665f5..1d60575 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -368,7 +368,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 public Segments calculateToBeSegments(CubeSegment newSegment) {
-return segments.calculateToBeSegments(newSegment, 
getModel().getPartitionDesc().isPartitioned());
+return segments.calculateToBeSegments(newSegment);
 }
 
 public CubeSegment getLastSegment() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 5339d81..e155f86 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -361,7 +361,8 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 return Segments.sourceOffsetContains(this, seg);
 }
 
-public void validate() {
+@Override
+public void validate() throws IllegalStateException {
 if 
(cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
 if (!isSourceOffsetsOn() && dateRangeStart >= dateRangeEnd)
 throw new IllegalStateException("Invalid segment, 
dateRangeStart(" + dateRangeStart + ") must be smaller than dateRangeEnd(" + 
dateRangeEnd + ") in segment " + this);

http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index f006613..d46ea96 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -37,4 +37,7 @@ public interface ISegment {
 public SegmentStatusEnum getStatus();
 
 public long getLastBuildTime();
+
+public void validate() throws IllegalStateException;
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 5198dc7..9371f76 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -229,7 +229,7 @@ public class Segments extends 
ArrayList {
  * - Favors new segments over the old
  * - Favors big segments over the small
  */
-public Segments calculateToBeSegments(ISegment newSegment, boolean 
isPartitioned) {
+public Segments calculateToBeSegments(ISegment newSegment) {
 
 Segments tobe = (Segments) this.clone();
 if (newSegment != null && !tobe.contains(newSegment)) {
@@ -242,12 +242,12 @@ public class Segments extends 
ArrayList {

[kylin] Git Push Summary

2016-12-05 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYILN-1875-B [deleted] 15af96402

kylin git commit: minor sql fix, CI passed

2016-12-05 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1875 77fa237ff -> 706d6137f


minor sql fix, CI passed


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/706d6137
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/706d6137
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/706d6137

Branch: refs/heads/KYLIN-1875
Commit: 706d6137ff96ff78feb19487442725d087064571
Parents: 77fa237
Author: Yang Li 
Authored: Tue Dec 6 07:22:03 2016 +0800
Committer: Yang Li 
Committed: Tue Dec 6 07:22:03 2016 +0800

--
 kylin-it/src/test/resources/query/sql_subquery/query02.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/706d6137/kylin-it/src/test/resources/query/sql_subquery/query02.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql 
b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
index 81b4887..e6751b7 100644
--- a/kylin-it/src/test/resources/query/sql_subquery/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
@@ -18,7 +18,7 @@
 
 SELECT
   week_beg_dt
-  ,sum(price)
+  ,sum(price) as sum_price
 FROM
 ( 
   select

[1/8] kylin git commit: KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc

2016-12-05 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYILN-1875-B [created] 15af96402


KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d3a11ffd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d3a11ffd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d3a11ffd

Branch: refs/heads/KYILN-1875-B
Commit: d3a11ffdd1672d271476d712669385498fba3f6b
Parents: 5460dea
Author: Li Yang 
Authored: Thu Dec 1 14:10:44 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:38:48 2016 +0800

--
 .../apache/kylin/cube/RawQueryLastHacker.java   |  8 +---
 .../metadata/measure/MeasureCodecTest.java  |  3 +-
 .../kylin/measure/raw/RawMeasureType.java   | 12 ++---
 .../kylin/metadata/model/FunctionDesc.java  | 26 ---
 .../kylin/metadata/model/ParameterDesc.java | 47 +++-
 .../apache/kylin/storage/StorageMockUtils.java  | 32 +++--
 .../kylin/query/relnode/OLAPAggregateRel.java   | 12 ++---
 7 files changed, 59 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d3a11ffd/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index b0a4823..682e48c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -57,12 +57,8 @@ public class RawQueryLastHacker {
 sqlDigest.groupbyColumns.add(col);
 } else {
 // For measure columns, take them as metric columns with 
aggregation function SUM().
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc sumFunc = new FunctionDesc();
-sumFunc.setExpression("SUM");
-sumFunc.setParameter(colParameter);
+ParameterDesc parameter = ParameterDesc.newInstance(col);
+FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", 
parameter, null);
 
 boolean measureHasSum = false;
 for (MeasureDesc colMeasureDesc : cubeDesc.getMeasures()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/d3a11ffd/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 6bb71e9..18680ec 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -83,8 +83,7 @@ public class MeasureCodecTest extends 
LocalFileMetadataTestCase {
 
 private MeasureDesc measure(String returnType) {
 MeasureDesc desc = new MeasureDesc();
-FunctionDesc func = new FunctionDesc();
-func.setReturnType(returnType);
+FunctionDesc func = FunctionDesc.newInstance(null, null, returnType);
 desc.setFunction(func);
 return desc;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d3a11ffd/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 3a49d31..a5bb06b 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -197,18 +197,12 @@ public class RawMeasureType extends 
MeasureType {
 if (sqlDigest.isRawQuery) {
 for (MeasureDesc measureDesc : measureDescs) {
 TblColRef col = this.getRawColumn(measureDesc.getFunction());
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc rawFunc = new FunctionDesc();
-rawFunc.setExpression("RAW");
-rawFunc.setParameter(colParameter);
+ParameterDesc colParameter = ParameterDesc.newInstance(col);
+FunctionDesc rawFunc =

[5/8] kylin git commit: KYLIN-1875 Support measure on non-root-fact-table

2016-12-05 Thread liyang

KYLIN-1875 Support measure on non-root-fact-table


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/104240d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/104240d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/104240d0

Branch: refs/heads/KYILN-1875-B
Commit: 104240d0f5b5f32f62595ca1caae03a4303e1c30
Parents: d3a11ff
Author: Li Yang 
Authored: Thu Dec 1 18:59:08 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:38:49 2016 +0800

--
 .../kylin/metadata/project/ProjectL2Cache.java  | 26 +---
 .../template/cube_desc/kylin_sales_cube.json| 18 +++---
 .../kylin/query/relnode/OLAPTableScan.java  |  4 +++
 3 files changed, 30 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/104240d0/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 82c0de3..14d7843 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
@@ -126,17 +127,17 @@ class ProjectL2Cache {
 return Collections.unmodifiableSet(tableCache.realizations);
 }
 
-public List listEffectiveRewriteMeasures(String project, 
String factTable, boolean onlyRewriteMeasure) {
-Set realizations = getRealizationsByTable(project, 
factTable);
+public List listEffectiveRewriteMeasures(String project, 
String table, boolean onlyRewriteMeasure) {
+Set realizations = getRealizationsByTable(project, 
table);
 List result = Lists.newArrayList();
 for (IRealization r : realizations) {
-if (r.getModel().isFactTable(factTable) && r.isReady()) {
-for (MeasureDesc m : r.getMeasures()) {
-FunctionDesc func = m.getFunction();
-if (onlyRewriteMeasure) {
-if (func.needRewrite())
-result.add(m);
-} else {
+if (!r.isReady())
+continue;
+
+for (MeasureDesc m : r.getMeasures()) {
+FunctionDesc func = m.getFunction();
+if (belongToTable(func, table, r.getModel())) {
+if (!onlyRewriteMeasure || func.needRewrite()) {
 result.add(m);
 }
 }
@@ -145,6 +146,13 @@ class ProjectL2Cache {
 return result;
 }
 
+private boolean belongToTable(FunctionDesc func, String table, 
DataModelDesc model) {
+// measure belong to the first column parameter's table
+List cols = func.getParameter().getColRefs();
+String belongTo = cols.isEmpty() ? 
model.getRootFactTable().getTableIdentity() : cols.get(0).getTable();
+return belongTo.equals(table);
+}
+
 // 

 // build the cache
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/104240d0/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 48eef46..2a27305 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -103,26 +103,26 @@
   "returntype" : "decimal(19,4)"
 }
   }, {
-"name" : "GMV_MIN",
+"name" : "BUYER_LEVEL_SUM",
 "function" : {
-  "expression" : "MIN",
+  "expression" : "SUM",
   "parameter" : {
 "type" : "column",
-"value" : "PRICE",
+"value" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
 "next_parameter" : null
   },
-  "returntype" : "decimal(19,4)"
+  "returntype" : "bigint"
 }
   }, {
-"name" : "GMV_MAX",
+"name" : "SELLER_LEVEL_SUM",
 "function" : {
-  "expression" : "MAX",
+

[7/8] kylin git commit: minor, test query of group by pushdown

2016-12-05 Thread liyang

minor, test query of group by pushdown


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef53fa40
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef53fa40
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef53fa40

Branch: refs/heads/KYILN-1875-B
Commit: ef53fa40a07167d207f968d8003069d5482f3770
Parents: 3d706ef
Author: Yang Li 
Authored: Sat Dec 3 12:33:14 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:38:50 2016 +0800

--
 .../resources/query/sql_subquery/query02.sql| 31 
 .../query/sql_subquery/query02.sql.disable  | 25 
 2 files changed, 31 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef53fa40/kylin-it/src/test/resources/query/sql_subquery/query02.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql 
b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
new file mode 100644
index 000..81b4887
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+  week_beg_dt
+  ,sum(price)
+FROM
+( 
+  select
+test_cal_dt.week_beg_dt
+,test_kylin_fact.price
+  from test_kylin_fact 
+  inner join edw.test_cal_dt as test_cal_dt
+on test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+) t
+group by week_beg_dt 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef53fa40/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disable
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disable 
b/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disable
deleted file mode 100644
index 968dbae..000
--- a/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disable
+++ /dev/null
@@ -1,25 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT sum(1) AS "COL" 
- FROM ( 
- select test_cal_dt.week_beg_dt, sum(test_kylin_fact.price) as sum_price
- from test_kylin_fact 
- inner join test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
- group by test_cal_dt.week_beg_dt 
- ) "TableauSQL"

[3/8] kylin git commit: KYLIN-1875 Normalize table and column names in JSON

2016-12-05 Thread liyang

KYLIN-1875 Normalize table and column names in JSON


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e0713d34
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e0713d34
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e0713d34

Branch: refs/heads/KYILN-1875-B
Commit: e0713d348649e4a9c8abd2c1d1f47e08929a7660
Parents: fc1e11a
Author: Li Yang 
Authored: Tue Nov 29 20:32:04 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:38:48 2016 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   | 22 +---
 .../org/apache/kylin/cube/model/CubeDesc.java   |  1 +
 .../apache/kylin/cube/model/DimensionDesc.java  | 19 +-
 .../model/validation/rule/FunctionRule.java | 37 
 .../kylin/metadata/model/DataModelDesc.java | 13 ++-
 .../kylin/metadata/model/FunctionDesc.java  |  5 +--
 .../metadata/model/ModelDimensionDesc.java  | 19 +++---
 .../kylin/metadata/model/PartitionDesc.java |  4 +--
 8 files changed, 64 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 677b713..011035b 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -81,8 +81,7 @@ public class FactTableGenerator {
 // table(appear as fk in fact table)
 TreeMap lookupTableKeys = new 
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
-// possible values of lookupTableKeys, extracted from existing lookup
-// tables.
+// possible values of lookupTableKeys, extracted from existing lookup 
tables.
 // The key is in the format of tablename/columnname
 TreeMap feasibleValues = new 
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
@@ -244,7 +243,7 @@ public class FactTableGenerator {
 JoinDesc join = dim.getJoin();
 if (join != null) {
 String lookupTable = dim.getTableRef().getTableIdentity();
-for (String column : join.getPrimaryKey()) {
+for (String column : dropAlias(join.getPrimaryKey())) {
 if (!lookupTableKeys.containsKey(lookupTable)) {
 lookupTableKeys.put(lookupTable, new 
LinkedList());
 }
@@ -297,8 +296,8 @@ public class FactTableGenerator {
 for (DimensionDesc dim : dimensions) {
 JoinDesc jDesc = dim.getJoin();
 if (jDesc != null) {
-String[] fks = jDesc.getForeignKey();
-String[] pks = jDesc.getPrimaryKey();
+String[] fks = dropAlias(jDesc.getForeignKey());
+String[] pks = dropAlias(jDesc.getPrimaryKey());
 int num = fks.length;
 for (int i = 0; i < num; ++i) {
 String value = dim.getTableRef().getTableIdentity() + "/" 
+ pks[i];
@@ -343,6 +342,19 @@ public class FactTableGenerator {
 return createTable(this.rowCount, factTableCol2LookupCol, 
lookupCol2factTableCol, usedCols);
 }
 
+private String[] dropAlias(String[] aliasDotCol) {
+String[] result = new String[aliasDotCol.length];
+for (int i = 0; i < aliasDotCol.length; i++) {
+String str = aliasDotCol[i];
+int cut = str.lastIndexOf('.');
+if (cut >= 0) {
+str = str.substring(cut + 1);
+}
+result[i] = str;
+}
+return result;
+}
+
 private String normToTwoDigits(int v) {
 if (v < 10)
 return "0" + v;

http://git-wip-us.apache.org/repos/asf/kylin/blob/e0713d34/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 853571c..f95cceb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -543,6 +543,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 initMeasureColumns();
 
 rowkey.init(this);
+
 validateAggregationGroups(); // check if aggregation group is valid
 for (AggregationGroup agg : this.aggregationGroups) {

[4/8] kylin git commit: KYLIN-1875 fix PK-FK derive from each other

2016-12-05 Thread liyang

KYLIN-1875 fix PK-FK derive from each other


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/93003b1c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/93003b1c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/93003b1c

Branch: refs/heads/KYILN-1875-B
Commit: 93003b1ca34255766c8912b4e4b39b2af8964dfa
Parents: 104240d
Author: Yang Li 
Authored: Thu Dec 1 21:42:52 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:38:49 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  8 +--
 .../org/apache/kylin/cube/model/CubeDesc.java   | 52 +++-
 .../storage/gtrecord/CubeTupleConverter.java|  2 +-
 .../gtrecord/GTCubeStorageQueryBase.java|  2 +-
 .../translate/DerivedFilterTranslator.java  |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  2 +-
 .../query/enumerator/LookupTableEnumerator.java |  2 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  2 +-
 .../hbase/cube/v1/CubeTupleConverter.java   |  2 +-
 9 files changed, 41 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/93003b1c/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 0c80c07..139f37a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -44,7 +44,6 @@ import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -54,6 +53,7 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TableDesc;
@@ -629,10 +629,10 @@ public class CubeManager implements IRealizationProvider {
 cubeMap.removeLocal(cubeName);
 }
 
-public LookupStringTable getLookupTable(CubeSegment cubeSegment, 
DimensionDesc dim) {
+public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc 
join) {
 
-String tableName = dim.getTableRef().getTableIdentity();
-String[] pkCols = dim.getJoin().getPrimaryKey();
+String tableName = join.getPKSide().getTableIdentity();
+String[] pkCols = join.getPrimaryKey();
 String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
 if (snapshotResPath == null)
 throw new IllegalStateException("No snaphot for table '" + 
tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() 
+ "/" + cubeSegment);

http://git-wip-us.apache.org/repos/asf/kylin/blob/93003b1c/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f95cceb..f8c316c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -101,20 +102,20 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public static class DeriveInfo {
 public DeriveType type;
-public DimensionDesc dimension;
+public JoinDesc join;
 public TblColRef[] columns;
 public boolean isOneToOne; // only used when ref from derived to host
 
-DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] 
columns, boolean isOneToOne) {
+DeriveInfo(DeriveType type, JoinDesc join,

[8/8] kylin git commit: minor minor

2016-12-05 Thread liyang

minor minor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/15af9640
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/15af9640
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/15af9640

Branch: refs/heads/KYILN-1875-B
Commit: 15af964023e091f01eca553d8c60f86ad89f218f
Parents: ef53fa4
Author: Yang Li 
Authored: Mon Dec 5 19:53:40 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:38:50 2016 +0800

--
 .../java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java   | 1 +
 .../main/java/org/apache/kylin/metadata/model/DataModelDesc.java   | 2 +-
 .../src/main/java/org/apache/kylin/metadata/model/TblColRef.java   | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/15af9640/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index c8a8f07..ecb8e61 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -121,6 +121,7 @@ public class ColumnTupleFilter extends TupleFilter {
 }
 }
 
+@SuppressWarnings("deprecation")
 @Override
 public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/15af9640/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 898ff74..3f868a2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -112,7 +112,7 @@ public class DataModelDesc extends RootPersistentEntity {
 return name;
 }
 
-// for test only
+// for test mainly
 @Deprecated
 public void setName(String name) {
 this.name = name;

http://git-wip-us.apache.org/repos/asf/kylin/blob/15af9640/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index bf8d36b..5d72c3f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -84,7 +84,7 @@ public class TblColRef implements Serializable {
 col.table = tableRef;
 }
 
-// for test only
+// for test mainly
 public static TblColRef mockup(TableDesc table, int oneBasedColumnIndex, 
String name, String datatype) {
 ColumnDesc desc = new ColumnDesc();
 String id = "" + oneBasedColumnIndex;

[6/8] kylin git commit: minor, rollback ITKylinQueryTest

2016-12-05 Thread liyang

minor, rollback ITKylinQueryTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3d706efa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3d706efa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3d706efa

Branch: refs/heads/KYILN-1875-B
Commit: 3d706efa5e56500e25cd233ec5a9cb435ef44fe9
Parents: 93003b1
Author: Yang Li 
Authored: Fri Dec 2 06:32:25 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:38:50 2016 +0800

--
 .../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3d706efa/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 6d91753..90324b5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -48,7 +48,7 @@ import org.junit.rules.ExpectedException;
 
 import com.google.common.collect.Maps;
 
-//@Ignore("KylinQueryTest is contained by ITCombinationTest")
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
 public class ITKylinQueryTest extends KylinTestBase {
 
 @Rule

[2/2] kylin git commit: KYLIN-2245 code review

2016-12-05 Thread liyang

KYLIN-2245 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fc1e11aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fc1e11aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fc1e11aa

Branch: refs/heads/master
Commit: fc1e11aa8a2d20add72e382396787d9cb09771da
Parents: 0a441c3
Author: Yang Li 
Authored: Mon Dec 5 21:20:21 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:20:21 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeInstance.java| 4 
 .../src/main/java/org/apache/kylin/cube/CubeManager.java | 2 +-
 .../main/java/org/apache/kylin/metadata/model/ISegment.java  | 4 ++--
 .../main/java/org/apache/kylin/metadata/model/Segments.java  | 8 
 4 files changed, 7 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index ecbb437..a3665f5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -371,10 +371,6 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return segments.calculateToBeSegments(newSegment, 
getModel().getPartitionDesc().isPartitioned());
 }
 
-public Pair 
findMergeOffsetsByDateRange(Segments segs, long startDate, long 
endDate, long skipSegDateRangeCap) {
-return this.segments.findMergeOffsetsByDateRange(segs, startDate, 
endDate, skipSegDateRangeCap);
-}
-
 public CubeSegment getLastSegment() {
 List existing = getSegments();
 if (existing.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 296a4e7..0c80c07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -519,7 +519,7 @@ public class CubeManager implements IRealizationProvider {
 if (isOffsetsOn) {
 // offset cube, merge by date range?
 if (startOffset == endOffset) {
-Pair pair = 
cube.findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), 
startDate, endDate, Long.MAX_VALUE);
+Pair pair = 
cube.getSegments(SegmentStatusEnum.READY).findMergeOffsetsByDateRange(startDate,
 endDate, Long.MAX_VALUE);
 if (pair == null)
 throw new IllegalArgumentException("Find no segments to 
merge by date range " + startDate + "-" + endDate + " for cube " + cube);
 startOffset = pair.getFirst().getSourceOffsetStart();

http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index 9d26927..f006613 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -26,6 +26,8 @@ public interface ISegment {
 
 public long getDateRangeEnd();
 
+public boolean isSourceOffsetsOn();
+
 public long getSourceOffsetStart();
 
 public long getSourceOffsetEnd();
@@ -35,6 +37,4 @@ public interface ISegment {
 public SegmentStatusEnum getStatus();
 
 public long getLastBuildTime();
-
-public boolean isSourceOffsetsOn();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index bc115cc..5198dc7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -187,8 +187,8 @@ public class Segments extends 
ArrayList {

[1/2] kylin git commit: KYLIN-2245 slim Segments in CubeMananger

2016-12-05 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 59a30f66d -> fc1e11aa8


KYLIN-2245 slim Segments in CubeMananger

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0a441c3f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0a441c3f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0a441c3f

Branch: refs/heads/master
Commit: 0a441c3fa30aee0a06cfc6301f7fbfa412103179
Parents: 59a30f6
Author: Cheng Wang 
Authored: Mon Dec 5 16:17:31 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 20:20:30 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |  14 ++
 .../java/org/apache/kylin/cube/CubeManager.java | 165 +--
 .../apache/kylin/metadata/model/ISegment.java   |   5 +-
 .../apache/kylin/metadata/model/Segments.java   | 206 ++-
 4 files changed, 221 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 8b12c2e..ecbb437 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.cube;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Set;
 
@@ -25,6 +26,7 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
@@ -361,6 +363,18 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return this.getDescriptor().getAutoMergeTimeRanges() != null && 
this.getDescriptor().getAutoMergeTimeRanges().length > 0;
 }
 
+public Pair autoMergeCubeSegments() throws IOException {
+return segments.autoMergeCubeSegments(needAutoMerge(), getName(), 
getDescriptor().getAutoMergeTimeRanges());
+}
+
+public Segments calculateToBeSegments(CubeSegment newSegment) {
+return segments.calculateToBeSegments(newSegment, 
getModel().getPartitionDesc().isPartitioned());
+}
+
+public Pair 
findMergeOffsetsByDateRange(Segments segs, long startDate, long 
endDate, long skipSegDateRangeCap) {
+return this.segments.findMergeOffsetsByDateRange(segs, startDate, 
endDate, skipSegDateRangeCap);
+}
+
 public CubeSegment getLastSegment() {
 List existing = getSegments();
 if (existing.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4ba29af..296a4e7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -520,7 +519,7 @@ public class CubeManager implements IRealizationProvider {
 if (isOffsetsOn) {
 // offset cube, merge by date range?
 if (startOffset == endOffset) {
-Pair pair = 
findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), 
startDate, endDate, Long.MAX_VALUE);
+Pair pair = 
cube.findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), 
startDate, endDate, Long.MAX_VALUE);
 if (pair == null)
 throw new IllegalArgumentException("Find no segments to 
merge by date range " + startDate + "-" + endDate + " for cube " + cube);
 startOffset = pair.getFirst().getSourceOffsetStart();
@@ -580,32 +579,6 @@ public class CubeManager implements IRealizationProvider {
 return newSegment;
 }
 
-private Pair 
findMergeOffsetsByDateRange(List segments, long startDate, long 
endDate,

kylin git commit: KYLIN-2246 redesign the way to decide layer cubing reducer count

2016-12-05 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2246 [created] 815887e73


KYLIN-2246 redesign the way to decide layer cubing reducer count


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/815887e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/815887e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/815887e7

Branch: refs/heads/KYLIN-2246
Commit: 815887e73a5c3b0852b6cf5650400235797d7ce9
Parents: 59a30f6
Author: Hongbin Ma 
Authored: Mon Dec 5 21:02:36 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 5 21:02:44 2016 +0800

--
 .../kylin/cube/cuboid/CuboidScheduler.java  | 31 +++-
 .../kylin/engine/mr/common/CubeStatsReader.java | 26 ++-
 .../apache/kylin/engine/mr/steps/CuboidJob.java | 52 +
 .../engine/mr/steps/LayerReduerNumSizing.java   | 82 
 .../kylin/engine/mr/steps/MergeCuboidJob.java   |  2 +-
 5 files changed, 138 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/815887e7/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index bd6a37a..733aded 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -18,7 +18,7 @@
 
 package org.apache.kylin.cube.cuboid;
 
-/** 
+/**
  */
 
 import java.util.Collections;
@@ -31,6 +31,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeDesc;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
@@ -39,6 +40,7 @@ public class CuboidScheduler {
 private final CubeDesc cubeDesc;
 private final long max;
 private final Map cache;
+private List cuboidsByLayer;
 
 public CuboidScheduler(CubeDesc cubeDesc) {
 this.cubeDesc = cubeDesc;
@@ -232,4 +234,31 @@ public class CuboidScheduler {
 getSubCuboidIds(cuboidId, result);
 }
 }
+
+public List getCuboidsByLayer() {
+if (cuboidsByLayer != null) {
+return cuboidsByLayer;
+}
+
+int totalNum = 0;
+int layerNum = cubeDesc.getBuildLevel();
+cuboidsByLayer = Lists.newArrayList();
+
+
cuboidsByLayer.add(Collections.singletonList(Cuboid.getBaseCuboidId(cubeDesc)));
+totalNum++;
+
+for (int i = 1; i <= layerNum; i++) {
+List lastLayer = cuboidsByLayer.get(i - 1);
+List newLayer = Lists.newArrayList();
+for (Long parent : lastLayer) {
+newLayer.addAll(getSpanningCuboid(parent));
+}
+cuboidsByLayer.add(newLayer);
+totalNum += newLayer.size();
+}
+
+int size = getAllCuboidIds().size();
+Preconditions.checkState(totalNum == size, "total Num: " + totalNum + 
" actual size: " + size);
+return cuboidsByLayer;
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/815887e7/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c917cfb..1cf5da6 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -29,6 +29,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BytesWritable;
@@ -75,9 +76,11 @@ public class CubeStatsReader {
 final int mapperNumberOfFirstBuild; // becomes meaningless after merge
 final double mapperOverlapRatioOfFirstBuild; // becomes meaningless after 
merge
 final Map cuboidRowEstimatesHLL;
+final CuboidScheduler cuboidScheduler;
 
 public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) 
throws IOException {
 ResourceStore store = ResourceStore.getStore(kylinConfig);
+cuboidScheduler = new CuboidScheduler(cubeSegment.getCubeDesc());
 String statsKey = cubeSegment.getStatisticsResourcePath();

kylin git commit: KYLIN-2248 TopN merge further optimization after KYLIN-1917

2016-12-05 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master af429e5cb -> 59a30f66d


KYLIN-2248 TopN merge further optimization after KYLIN-1917


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59a30f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59a30f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59a30f66

Branch: refs/heads/master
Commit: 59a30f66d47cc1838e6852405699fd7957bfac29
Parents: af429e5
Author: shaofengshi 
Authored: Sun Dec 4 09:39:45 2016 +0800
Committer: shaofengshi 
Committed: Mon Dec 5 17:42:34 2016 +0800

--
 .../apache/kylin/measure/topn/TopNCounter.java  | 47 ++--
 1 file changed, 13 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59a30f66/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index 968e694..caf7961 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -26,11 +26,9 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-import com.google.common.collect.Maps;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 
 /**
  * Modified from the StreamSummary.java in 
https://github.com/addthis/stream-lib
@@ -157,41 +155,22 @@ public class TopNCounter implements 
Iterable {
  * @return
  */
 public TopNCounter merge(TopNCounter another) {
-double m1 = 0.0, m2 = 0.0;
-if (this.size() >= this.capacity) {
-m1 = this.counterList.getLast().count;
-}
-
-if (another.size() >= another.capacity) {
-m2 = another.counterList.getLast().count;
-}
-
-Set duplicateItems = Sets.newHashSet();
-List notDuplicateItems = Lists.newArrayList();
-
-for (Map.Entry entry : this.counterMap.entrySet()) {
-T item = entry.getKey();
-Counter existing = another.counterMap.get(item);
-if (existing != null) {
-duplicateItems.add(item);
-} else {
-notDuplicateItems.add(item);
+boolean thisFull = this.size() >= this.capacity;
+boolean anotherFull = another.size() >= another.capacity;
+double m1 = thisFull ? this.counterList.getLast().count : 0.0;
+double m2 = anotherFull ? another.counterList.getLast().count : 0.0;
+
+if (anotherFull == true) {
+for (Counter entry : this.counterMap.values()) {
+entry.count += m2;
 }
 }
 
-for (T item : duplicateItems) {
-this.offer(item, another.counterMap.get(item).count);
-}
-
-for (T item : notDuplicateItems) {
-this.offer(item, m2);
-}
-
 for (Map.Entry entry : another.counterMap.entrySet()) {
-T item = entry.getKey();
-if (duplicateItems.contains(item) == false) {
-double counter = entry.getValue().count;
-this.offer(item, counter + m1);
+if (this.counterMap.containsKey(entry.getKey())) {
+this.offer(entry.getValue().getItem(), (entry.getValue().count 
- m2));
+} else {
+this.offer(entry.getValue().getItem(), entry.getValue().count 
+ m1);
 }
 }

[3/5] kylin git commit: KYLIN-1875 Rollback sample data back to star schema

[2/5] kylin git commit: KYLIN-1875 Rollback sample data back to star schema

[5/5] kylin git commit: KYLIN-1875 Rollback sample data back to star schema

[4/5] kylin git commit: KYLIN-1875 Rollback sample data back to star schema

[kylin] Git Push Summary

[kylin] Git Push Summary

kylin git commit: refine sample cube, add kind=LOOKUP attr

svn commit: r1772835 - in /kylin/site: community/index.html docs/howto/howto_use_restapi.html docs15/howto/howto_use_restapi.html docs16/howto/howto_use_restapi.html feed.xml

[kylin] Git Push Summary

[kylin] Git Push Summary

[2/3] kylin git commit: minor, generate item_count value for sample data

[3/3] kylin git commit: minor, generate item_count value for sample data

kylin git commit: update committer info

kylin git commit: Thanks chitre.a...@gmail.com: typo in rest api doc

kylin git commit: KYLIN-2245 further minor refactor

[kylin] Git Push Summary

kylin git commit: minor sql fix, CI passed

[1/8] kylin git commit: KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc

[5/8] kylin git commit: KYLIN-1875 Support measure on non-root-fact-table

[7/8] kylin git commit: minor, test query of group by pushdown

[3/8] kylin git commit: KYLIN-1875 Normalize table and column names in JSON

[4/8] kylin git commit: KYLIN-1875 fix PK-FK derive from each other

[8/8] kylin git commit: minor minor

[6/8] kylin git commit: minor, rollback ITKylinQueryTest

[2/2] kylin git commit: KYLIN-2245 code review

[1/2] kylin git commit: KYLIN-2245 slim Segments in CubeMananger

kylin git commit: KYLIN-2246 redesign the way to decide layer cubing reducer count

kylin git commit: KYLIN-2248 TopN merge further optimization after KYLIN-1917

28 matches

Site Navigation

Mail list logo

Footer information