[24/27] kylin git commit: KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc

[liyang]

KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47e922e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47e922e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47e922e3

Branch: refs/heads/KYLIN-1875
Commit: 47e922e32976c30390712a2ead554c8b9952
Parents: e6a1b23
Author: Li Yang 
Authored: Thu Dec 1 14:10:44 2016 +0800
Committer: Yang Li 
Committed: Fri Dec 2 21:01:59 2016 +0800

--
 .../apache/kylin/cube/RawQueryLastHacker.java   |  8 +---
 .../metadata/measure/MeasureCodecTest.java  |  3 +-
 .../kylin/measure/raw/RawMeasureType.java   | 12 ++---
 .../kylin/metadata/model/FunctionDesc.java  | 26 ---
 .../kylin/metadata/model/ParameterDesc.java | 47 +++-
 .../apache/kylin/storage/StorageMockUtils.java  | 32 +++--
 .../kylin/query/relnode/OLAPAggregateRel.java   | 12 ++---
 7 files changed, 59 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/47e922e3/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index b0a4823..682e48c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -57,12 +57,8 @@ public class RawQueryLastHacker {
 sqlDigest.groupbyColumns.add(col);
 } else {
 // For measure columns, take them as metric columns with 
aggregation function SUM().
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc sumFunc = new FunctionDesc();
-sumFunc.setExpression("SUM");
-sumFunc.setParameter(colParameter);
+ParameterDesc parameter = ParameterDesc.newInstance(col);
+FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", 
parameter, null);
 
 boolean measureHasSum = false;
 for (MeasureDesc colMeasureDesc : cubeDesc.getMeasures()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/47e922e3/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 6bb71e9..18680ec 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -83,8 +83,7 @@ public class MeasureCodecTest extends 
LocalFileMetadataTestCase {
 
 private MeasureDesc measure(String returnType) {
 MeasureDesc desc = new MeasureDesc();
-FunctionDesc func = new FunctionDesc();
-func.setReturnType(returnType);
+FunctionDesc func = FunctionDesc.newInstance(null, null, returnType);
 desc.setFunction(func);
 return desc;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/47e922e3/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 3a49d31..a5bb06b 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -197,18 +197,12 @@ public class RawMeasureType extends 
MeasureType {
 if (sqlDigest.isRawQuery) {
 for (MeasureDesc measureDesc : measureDescs) {
 TblColRef col = this.getRawColumn(measureDesc.getFunction());
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc rawFunc = new FunctionDesc();
-rawFunc.setExpression("RAW");
-rawFunc.setParameter(colParameter);
+ParameterDesc colParameter = ParameterDesc.newInstance(col);
+FunctionDesc rawFunc = FunctionDesc.newInstance("RAW", 
colParameter, null);
 
 if 

[05/27] kylin git commit: KYLIN-2217 Reducers build dictionaries locally

[liyang]

KYLIN-2217 Reducers build dictionaries locally

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1af08e4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1af08e4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1af08e4b

Branch: refs/heads/KYLIN-1875
Commit: 1af08e4b8875d33bfc5dd124fed72d6042456c32
Parents: b1b90ad
Author: xiefan46 <958034...@qq.com>
Authored: Wed Nov 23 09:48:55 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 30 15:31:58 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   9 +-
 .../java/org/apache/kylin/cube/CubeManager.java |   3 +-
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  |  25 ++-
 .../apache/kylin/dict/DictionaryManager.java|  17 +-
 .../apache/kylin/dict/DictionaryProvider.java   |  28 
 .../dict/DictionaryReducerLocalGenerator.java   | 156 +++
 .../dict/IDictionaryReducerLocalBuilder.java|  31 
 .../kylin/dict/DictionaryProviderTest.java  | 109 +
 .../storage/translate/ColumnValueRange.java |   2 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  44 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 123 +--
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  54 +++
 .../storage/hbase/cube/v1/CubeStorageQuery.java |   6 +-
 13 files changed, 547 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7dcc771..766c04d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -719,7 +719,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 //UHC: ultra high cardinality columns, contain the ShardByColumns and the 
GlobalDictionaryColumns
 public int getUHCReducerCount() {
-return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
+return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
+}
+
+public boolean isReducerLocalBuildDict() {
+if (getUHCReducerCount() != 1) {
+return false;
+}
+return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
 }
 
 public String getYarnStatusCheckUrl() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b4422d2..119a21a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -213,6 +213,7 @@ public class CubeManager implements IRealizationProvider {
 return result;
 }
 
+
 public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable) throws IOException {
 CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
@@ -221,6 +222,7 @@ public class CubeManager implements IRealizationProvider {
 String builderClass = cubeDesc.getDictionaryBuilderClass(col);
 DictionaryInfo dictInfo = 
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, 
builderClass);
 
+
 saveDictionaryInfo(cubeSeg, col, dictInfo);
 return dictInfo;
 }
@@ -266,7 +268,6 @@ public class CubeManager implements IRealizationProvider {
 } catch (IOException e) {
 throw new IllegalStateException("Failed to get dictionary for cube 
segment" + cubeSeg + ", col" + col, e);
 }
-
 return (Dictionary) info.getDictionaryObject();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index a6aeb96..a4e1df0 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java

[18/27] kylin git commit: KYLIN-2245 refine CubeSegments

[liyang]

KYLIN-2245 refine CubeSegments

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ca53954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ca53954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ca53954

Branch: refs/heads/KYLIN-1875
Commit: 3ca53954415c62c2a91473c6d7bf831504b5e9db
Parents: 4a0ee79
Author: Cheng Wang 
Authored: Fri Dec 2 15:35:52 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 17:26:07 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java | 115 +++
 .../java/org/apache/kylin/cube/CubeManager.java |  10 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   8 +-
 .../apache/kylin/metadata/model/ISegment.java   |   6 +-
 .../apache/kylin/metadata/model/Segments.java   | 140 +++
 .../hbase/util/ExtendCubeToHybridCLI.java   |   4 +-
 .../kylin/tool/ExtendCubeToHybridCLI.java   |   4 +-
 7 files changed, 176 insertions(+), 111 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 061ab23..8b12c2e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,8 +18,6 @@
 
 package org.apache.kylin.cube;
 
-import java.util.ArrayList;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
@@ -34,6 +32,7 @@ import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
 import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
@@ -64,7 +63,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 cubeInstance.setName(cubeName);
 cubeInstance.setDescName(cubeDesc.getName());
 cubeInstance.setCreateTimeUTC(System.currentTimeMillis());
-cubeInstance.setSegments(new ArrayList());
+cubeInstance.setSegments(new Segments());
 cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
 cubeInstance.updateRandomUuid();
 
@@ -87,7 +86,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 
 @JsonManagedReference
 @JsonProperty("segments")
-private List segments = new ArrayList();
+private Segments segments = new Segments();
 
 @JsonProperty("create_time_utc")
 private long createTimeUTC;
@@ -97,39 +96,11 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 public List getBuildingSegments() {
-List buildingSegments = new ArrayList();
-if (null != segments) {
-for (CubeSegment segment : segments) {
-if (SegmentStatusEnum.NEW == segment.getStatus() || 
SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
-buildingSegments.add(segment);
-}
-}
-}
-
-return buildingSegments;
+return segments.getBuildingSegments();
 }
 
 public List getMergingSegments(CubeSegment mergedSegment) {
-LinkedList result = new LinkedList();
-if (mergedSegment == null)
-return result;
-
-for (CubeSegment seg : this.segments) {
-if (seg.getStatus() != SegmentStatusEnum.READY && seg.getStatus() 
!= SegmentStatusEnum.READY_PENDING)
-continue;
-
-if (seg == mergedSegment)
-continue;
-
-if (mergedSegment.sourceOffsetContains(seg)) {
-// make sure no holes
-if (result.size() > 0 && result.getLast().getSourceOffsetEnd() 
!= seg.getSourceOffsetStart())
-throw new IllegalStateException("Merging segments must not 
have holes between " + result.getLast() + " and " + seg);
-
-result.add(seg);
-}
-}
-return result;
+return segments.getMergingSegments(mergedSegment);
 }
 
 public CubeDesc getDescriptor() {
@@ -154,7 +125,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // in a temporary broken state, so that user can edit and fix it. Broken 
state is often due to
 // schema 

[12/27] kylin git commit: KYLIN-2213 add test query

[liyang]

KYLIN-2213 add test query


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3091f06a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3091f06a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3091f06a

Branch: refs/heads/KYLIN-1875
Commit: 3091f06a68e0b97b35639cbb65acbf532c114ca4
Parents: 9fc8f5e
Author: Li Yang 
Authored: Fri Dec 2 13:21:54 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 13:21:54 2016 +0800

--
 .../test/resources/query/sql_like/query20.sql   | 31 
 1 file changed, 31 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3091f06a/kylin-it/src/test/resources/query/sql_like/query20.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_like/query20.sql 
b/kylin-it/src/test/resources/query/sql_like/query20.sql
new file mode 100644
index 000..6101cb0
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query20.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select lstg_format_name as lstg_format_name, count(*) as cnt 
+ 
+ from test_kylin_fact 
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ 
+ 
+where lstg_format_name not like '%BIN%'
+group by lstg_format_name
\ No newline at end of file

[15/27] kylin git commit: KYLIN-2192 More Robust Global Dictionary

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
--
diff --git 
a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
 
b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
index caa8754..949599d 100644
--- 
a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
+++ 
b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
@@ -1,402 +1,402 @@
-2013-03-31,48028,0,\N,\N,\N,Auction,12,184.21,1001,1,Ebay
-2013-11-12,164262,0,\N,\N,\N,Others,5,172.03,1002,1,Ebay
-2013-04-06,82494,15,BookMagazines,NULL,Comic 
Books,Auction,14,66.6,1003,1,Ebay
-2013-05-17,66767,15,Home & Garden,NULL,Dogs,Auction,12,92.98,1004,1,Ebay
-2013-05-20,152801,0,Jewelry & 
Watches,NULL,Earrings,FP-GTC,5,132.33,1005,1,Ebay
-2013-06-16,43398,0,Home & Garden,NULL,Cheese & 
Crackers,FP-GTC,13,7.12,1006,1,Ebay
-2013-06-14,95173,0,Health & Beauty,Bath & Body,Bath Sets & 
Kits,Auction,14,204.28,1007,1,Ebay
-2013-03-22,158666,15,ToyHobbies,Action Figures,Anime & 
Manga,Auction,13,35.72,1008,1,Ebay
-2013-03-10,12688,0,eBay Premier,Books & Manuscripts,Books: 
Other,Auction,12,4.13,1009,1,Ebay
-2013-11-01,103324,15,ClothinShoeAccessories,Women's Shoes,Mixed 
Items,FP-GTC,5,27.48,1010,1,Ebay
-2013-06-16,108782,15,Vehicle Parts & Accessories,CaTruck Parts,Car Care & 
Cleaning,FP-GTC,14,9.26,1011,1,Ebay
-2013-09-12,80287,0,Computers/Tablets & Networking,Software,Office & 
Business,Auction,12,3.18,1012,1,Ebay
-2013-09-28,140746,100,eBay Motors,Parts & Accessories,Vintage Car & Truck 
Parts,Others,13,3.18,1013,1,Ebay
-2013-06-15,87118,0,Sporting Goods,Outdoor 
Sports,Paintball,ABIN,14,377.94,1014,1,Ebay
-2013-03-14,25147,0,Sports MeCards & Fan Shop,Fan Apparel & 
Souvenirs,Baseball-MLB,Auction,12,146.33,1015,1,Ebay
-2013-09-01,170302,15,Crafts,Embroidery,Design 
CDs,FP-GTC,5,51.23,1016,1,Ebay
-2013-05-29,53064,0,Business & Industrial,Heavy Equipment,Antique & Vintage 
Farm Equip,FP-non GTC,13,72.65,1017,1,Ebay
-2013-05-31,132939,0,Jewelry & Watches,Fashion 
Jewelry,Other,Auction,13,66.6,1018,1,Ebay
-2013-03-18,113593,15,Phones,Mobile Phones,Mobile 
Phones,Auction,12,9.26,1019,1,Ebay
-2013-07-19,34273,100,eBay Motors,Parts & 
Accessories,Motorcycle,Auction,14,583.44,1020,1,Ebay
-2013-06-23,106340,15,Home & Garden,Gardening,Hand 
Tools,FP-GTC,14,638.72,1021,1,Ebay
-2013-05-20,150265,15,Baby,Baby Clothing,Boys,FP-GTC,14,4.54,1022,1,Ebay
-2013-05-17,24760,0,Sports MeCards & Fan Shop,Fan Apparel & 
Souvenirs,Hockey-NHL,FP-GTC,12,319.79,1023,1,Ebay
-2013-03-11,37831,0,Collectibles,Advertising,Merchandise & 
Memorabilia,Auction,12,20.35,1024,1,Ebay
-2013-01-30,1120,3,Books,First Editions,Other,FP-non 
GTC,5,223.63,1025,1,Ebay
-2013-01-26,43972,100,eBay Motors,Parts & Accessories,ATV 
Parts,FP-GTC,13,204.28,1026,1,Ebay
-2013-03-22,166013,15,Computers,Computer Components & Parts,Video Capture & TV 
Tuner Cards,Auction,14,5.48,1027,1,Ebay
-2013-07-23,15568,15,Baby,Baby Clothing,Unisex,Auction,14,27.48,1028,1,Ebay
-2013-07-27,103178,15,ClothinShoeAccessories,Women's Bags,Women's 
Bags,FP-GTC,5,21.72,1029,1,Ebay
-2013-10-29,2023,0,Sporting Goods,Team 
Sports,Basketball,ABIN,12,3.18,1030,1,Ebay
-2013-10-08,94847,0,Consumer Electronics,Vehicle Electronics & GPS,Car 
Video,FP-GTC,11,491.32,1031,1,Ebay
-2013-04-26,15868,0,Real Estate,Land,Land,Auction,14,448.8,1032,1,Ebay
-2013-01-01,32876,0,Home & Garden,Home Improvement,Plumbing & 
Fixtures,Auction,13,415.73,1033,1,Ebay
-2013-01-15,62179,0,ClothinShoes & Accessories,Women's Clothing,Athletic 
Apparel,Auction,13,377.94,1034,1,Ebay
-2013-05-27,33038,15,Musical Instruments,Instruments,Guitars 
(Electric),FP-GTC,14,146.33,1035,1,Ebay
-2013-11-11,156614,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: 
RacinNASCAR,FP-GTC,5,7.12,1036,1,Ebay
-2013-03-08,106246,0,Health & Beauty,Hair Care & Styling,Shampoo & 
Conditioning,Auction,13,42.99,1037,1,Ebay
-2013-03-25,20865,0,ClothinShoes & Accessories,Men's Clothing,Athletic 
Apparel,Auction,13,12.85,1038,1,Ebay
-2013-08-20,15115,0,Video Games & Consoles,Video Games,Video 
Games,FP-GTC,13,55.89,1039,1,Ebay
-2013-05-17,3838,0,Jewelry & Watches,Fashion Jewelry,Charms & Charm 
Bracelets,FP-GTC,14,73.26,1040,1,Ebay
-2013-06-05,759,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & 
Vans,Auction,11,112.56,1041,1,Ebay
-2013-10-08,61323,0,Consumer Electronics,TVideo & Home Audio,TVideo & Audio 
Accessories,FP-non GTC,11,3.49,1042,1,Ebay
-2013-08-14,121153,0,Baby,Nursery Decor,Night 
Lights,Auction,13,184.21,1043,1,Ebay
-2013-08-14,88750,0,Consumer Electronics,Vehicle Electronics & GPS,Radar & 
Laser Detectors,Auction,13,157.14,1044,1,Ebay

[07/27] kylin git commit: org.apache.kylin.metadata.model.FunctionDesc#configuration should use default linkedhashmap to keep order

[liyang]

org.apache.kylin.metadata.model.FunctionDesc#configuration should use default 
linkedhashmap to keep order


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/93bf0d02
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/93bf0d02
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/93bf0d02

Branch: refs/heads/KYLIN-1875
Commit: 93bf0d0214fe446d794b61e02fb18ff8097611d1
Parents: 30cb1ac
Author: Hongbin Ma 
Authored: Thu Dec 1 11:20:53 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 11:40:25 2016 +0800

--
 .../apache/kylin/common/util/JacksonBean.java| 16 
 .../apache/kylin/common/util/JacksonTest.java| 19 +++
 .../kylin/metadata/model/FunctionDesc.java   |  8 
 3 files changed, 35 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
index 42357f2..6d1 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -23,6 +23,10 @@ import 
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class JacksonBean {
 
@@ -32,6 +36,10 @@ public class JacksonBean {
 @JsonInclude(JsonInclude.Include.NON_NULL)
 private int b;
 
+@JsonProperty("configuration")
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+private Map configuration = new HashMap();
+
 public String getA() {
 return a;
 }
@@ -48,6 +56,14 @@ public class JacksonBean {
 this.b = b;
 }
 
+public Map getConfiguration() {
+return configuration;
+}
+
+public void setConfiguration(Map configuration) {
+this.configuration = configuration;
+}
+
 @Override
 public String toString() {
 return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';

http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
index 81be7eb..0d7097a 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -18,22 +18,33 @@
 
 package org.apache.kylin.common.util;
 
-import java.io.IOException;
-
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 public class JacksonTest {
 @Test
 public void foo() throws IOException {
+HashMap a = new HashMap();
+a.put("1", "1");
+a.put("3", "3");
+a.put("2", "2");
+
+
 JacksonBean bean = new JacksonBean();
 bean.setA("valuea");
+bean.setConfiguration(a);
 
 String s = JsonUtil.writeValueAsString(bean);
 System.out.println(s);
 
-JacksonBean desBean = (JacksonBean) 
JsonUtil.readValue("{\"a\":\"valuea\"}", JacksonBean.class);
+JacksonBean desBean = (JacksonBean) 
JsonUtil.readValue("{\"a\":\"valuea\",\"b\":0,\"configuration\":{\"2\":\"2\",\"3\":\"3\",\"1\":\"1\"}}",
 JacksonBean.class);
+
 String x2 = JsonUtil.writeValueAsString(desBean);
-System.out.println(desBean);
 System.out.println(x2);
+
+System.out.println(desBean);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index ae7f805..7b41552 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 

[19/27] kylin git commit: KYLIN-2245 slim ISegment a bit

[liyang]

KYLIN-2245 slim ISegment a bit


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0eebd7d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0eebd7d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0eebd7d8

Branch: refs/heads/KYLIN-1875
Commit: 0eebd7d8ab10b6b2b4823d6a23eaa774ac36989e
Parents: 3ca5395
Author: Li Yang 
Authored: Fri Dec 2 17:23:16 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 17:26:13 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeSegment.java | 20 +++-
 .../apache/kylin/metadata/model/ISegment.java   |  1 -
 .../apache/kylin/metadata/model/Segments.java   | 12 +++-
 3 files changed, 14 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 1fc28eb..5339d81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -38,6 +38,7 @@ import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.ISegment;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;
 
@@ -350,29 +351,14 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 this.sourceOffsetEnd = sourceOffsetEnd;
 }
 
-public boolean dateRangeOverlaps(CubeSegment seg) {
-return dateRangeStart < seg.dateRangeEnd && seg.dateRangeStart < 
dateRangeEnd;
-}
-
-public boolean dateRangeContains(CubeSegment seg) {
-return dateRangeStart <= seg.dateRangeStart && seg.dateRangeEnd <= 
dateRangeEnd;
-}
-
 // date range is used in place of source offsets when offsets are missing
 public boolean sourceOffsetOverlaps(CubeSegment seg) {
-if (isSourceOffsetsOn())
-return sourceOffsetStart < seg.sourceOffsetEnd && 
seg.sourceOffsetStart < sourceOffsetEnd;
-else
-return dateRangeOverlaps(seg);
+return Segments.sourceOffsetOverlaps(this, seg);
 }
 
 // date range is used in place of source offsets when offsets are missing
-@Override
 public boolean sourceOffsetContains(ISegment seg) {
-if (isSourceOffsetsOn())
-return sourceOffsetStart <= ((CubeSegment) seg).sourceOffsetStart 
&& ((CubeSegment) seg).sourceOffsetEnd <= sourceOffsetEnd;
-else
-return dateRangeContains(((CubeSegment) seg));
+return Segments.sourceOffsetContains(this, seg);
 }
 
 public void validate() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index e97f4f4..e3fcdcb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -36,5 +36,4 @@ public interface ISegment{
 
 public long getLastBuildTime();
 
-public boolean sourceOffsetContains(ISegment seg);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 104c2af..f0a58cb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -21,6 +21,16 @@ package org.apache.kylin.metadata.model;
 import java.util.ArrayList;
 
 public class Segments extends ArrayList {
+
+private static final long serialVersionUID = 1L;
+
+public static boolean sourceOffsetContains(ISegment a, ISegment b) {
+return a.getSourceOffsetStart() <= b.getSourceOffsetStart() && 
b.getSourceOffsetEnd() <= a.getSourceOffsetEnd();
+}
+
+public static boolean sourceOffsetOverlaps(ISegment a, ISegment b) {
+return a.getSourceOffsetStart() < 

[09/27] kylin git commit: refine mapper and reducer log

[liyang]

refine mapper and reducer log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58224921
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58224921
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58224921

Branch: refs/heads/KYLIN-1875
Commit: 58224921d896e4479f5d034d43c044aacaf14200
Parents: 28ba1ea
Author: Hongbin Ma 
Authored: Thu Dec 1 18:15:46 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 18:15:46 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/KylinMapper.java   | 12 +---
 .../java/org/apache/kylin/engine/mr/KylinReducer.java  | 13 ++---
 .../apache/kylin/engine/mr/steps/CuboidReducer.java| 11 ---
 .../kylin/engine/mr/steps/HiveToBaseCuboidMapper.java  |  6 --
 .../kylin/engine/mr/steps/InMemCuboidMapper.java   |  7 +--
 .../kylin/engine/mr/steps/InMemCuboidReducer.java  | 10 +-
 .../apache/kylin/engine/mr/steps/NDCuboidMapper.java   | 10 --
 7 files changed, 33 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index a01f7a2..2b564e9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,6 +31,8 @@ import org.slf4j.LoggerFactory;
 public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
+protected int mapCounter = 0;
+
 protected void bindCurrentConfiguration(Configuration conf) {
 logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 HadoopUtil.setCurrentConfiguration(conf);
@@ -38,6 +41,9 @@ public class KylinMapper 
extends Mapper.Context context) throws IOException, InterruptedException {
 try {
+if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 
0) {
+logger.info("Accepting Mapper Key with ordinal: " + 
mapCounter);
+}
 doMap(key, value, context);
 } catch (IOException ex) { // KYLIN-2170
 logger.error("", ex);
@@ -53,11 +59,11 @@ public class KylinMapper 
extends Mapper.Context context) throws IOException, InterruptedException {
 super.map(key, value, context);
 }
-
+
 @Override
 final protected void cleanup(Mapper.Context context) throws IOException, InterruptedException {
 try {
@@ -76,7 +82,7 @@ public class KylinMapper 
extends Mapper.Context 
context) throws IOException, InterruptedException {
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index 2b63ce0..cb2d7a7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -29,7 +30,9 @@ import org.slf4j.LoggerFactory;
  */
 public class KylinReducer extends 
Reducer {
 private static final Logger logger = 

[04/27] kylin git commit: minor, massin udf reverse

[liyang]

minor, massin udf reverse

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1b90adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1b90adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1b90adb

Branch: refs/heads/KYLIN-1875
Commit: b1b90adbe896e808776caf49ad7172979cf860df
Parents: 875a593
Author: Roger Shi 
Authored: Wed Nov 30 13:35:12 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 30 13:49:16 2016 +0800

--
 .../metadata/filter/UDF/MassInTupleFilter.java  | 28 +++-
 1 file changed, 27 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b1b90adb/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 3f0546c..29c5550 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -50,6 +50,7 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 private String filterTableName;//key in MetadataManager.extFilterMap
 private String filterTableResourceIdentifier;//HDFS path, or hbase table 
name depending on FilterTableType
 private Functions.FilterTableType filterTableType;
+private boolean reverse = false;
 
 public MassInTupleFilter() {
 super(Lists. newArrayList(), 
TupleFilter.FilterOperatorEnum.MASSIN);
@@ -66,7 +67,18 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 valueProvider = 
VALUE_PROVIDER_FACTORY.getProvider(filterTableType, 
filterTableResourceIdentifier, column);
 }
 boolean ret = valueProvider.getMassInValues().contains(colValue);
-return ret;
+return reverse ? !ret : ret;
+}
+
+@Override
+public TupleFilter reverse() {
+try {
+MassInTupleFilter result = (MassInTupleFilter) this.clone();
+result.setReverse(!this.isReverse());
+return result;
+} catch (CloneNotSupportedException e) {
+throw new UnsupportedOperationException(e);
+}
 }
 
 @Override
@@ -144,4 +156,18 @@ public class MassInTupleFilter extends FunctionTupleFilter 
{
 return false;
 }
 
+public boolean isReverse() {
+return reverse;
+}
+
+public void setReverse(boolean reverse) {
+this.reverse = reverse;
+}
+
+@Override
+protected Object clone() throws CloneNotSupportedException {
+MassInTupleFilter result = new MassInTupleFilter();
+result.setReverse(this.isReverse());
+return result;
+}
 }

[01/27] kylin git commit: KYLIN-2195 could run get-properties.sh in source code dir [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1875 12230f4e3 -> 84ce3ad62 (forced update)


KYLIN-2195 could run get-properties.sh in source code dir


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8506e4d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8506e4d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8506e4d4

Branch: refs/heads/KYLIN-1875
Commit: 8506e4d4f75fbbda518997cac97e229fa5a0
Parents: bda8787
Author: lidongsjtu 
Authored: Tue Nov 29 18:24:07 2016 +0800
Committer: lidongsjtu 
Committed: Tue Nov 29 18:24:07 2016 +0800

--
 build/bin/get-properties.sh | 12 ++--
 1 file changed, 10 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8506e4d4/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 170442d..0fa4f6f 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -19,13 +19,21 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
+
 if [ $# != 1 ]
 then
 echo 'invalid input'
 exit -1
 fi
 
-job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
-tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
+if [[ $CI_MODE == 'true' ]]; then
+cd $dir
+job_jar=$(ls ../../assembly/target/kylin-*-job.jar)
+tool_jar=$(ls ../../tool/target/kylin-tool-*.jar|grep -v assembly)
+else
+job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
+tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
+fi
+
 result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 
2>/dev/null`
 echo "$result"

[03/27] kylin git commit: filter null value in cubedesc and model desc

[liyang]

filter null value in cubedesc and model desc

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/875a5931
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/875a5931
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/875a5931

Branch: refs/heads/KYLIN-1875
Commit: 875a5931ad8cad79cdec1e80ab9388ded9446091
Parents: 3186d17
Author: luguosheng <550175...@qq.com>
Authored: Wed Nov 30 10:51:51 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 30 13:48:29 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js  |  4 ++--
 webapp/app/js/controllers/modelEdit.js | 13 ++---
 webapp/app/js/utils/utils.js   | 24 
 3 files changed, 36 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/875a5931/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 888662d..0704a62 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -413,7 +413,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 
 if ($scope.isEdit) {
   CubeService.update({}, {
-cubeDescData: $scope.state.cubeSchema,
+cubeDescData: VdmUtil.filterNullValInObj($scope.state.cubeSchema),
 cubeName: $routeParams.cubeName,
 project: $scope.state.project
   }, function (request) {
@@ -453,7 +453,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   });
 } else {
   CubeService.save({}, {
-cubeDescData: $scope.state.cubeSchema,
+cubeDescData: VdmUtil.filterNullValInObj($scope.state.cubeSchema),
 project: $scope.state.project
   }, function (request) {
 if (request.successful) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/875a5931/webapp/app/js/controllers/modelEdit.js
--
diff --git a/webapp/app/js/controllers/modelEdit.js 
b/webapp/app/js/controllers/modelEdit.js
index 0b3db66..2f1b35e 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -19,7 +19,7 @@
 'use strict';
 
 
-KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, 
$location, $templateCache, $interpolate, MessageService, TableService, 
CubeDescService, ModelService, loadingRequest, 
SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager)
 {
+KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, 
$location, $templateCache, $interpolate, MessageService, TableService, 
CubeDescService, ModelService, loadingRequest, 
SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager,VdmUtil)
 {
 //add or edit ?
 var absUrl = $location.absUrl();
 $scope.modelMode = 
absUrl.indexOf("/models/add")!=-1?'addNewModel':absUrl.indexOf("/models/edit")!=-1?'editExistModel':'default';
@@ -188,7 +188,11 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, 
$routeParams, $locati
 loadingRequest.show();
 
 if ($scope.isEdit) {
-ModelService.update({}, 
{modelDescData:$scope.state.modelSchema, modelName: $routeParams.modelName, 
project: $scope.state.project}, function (request) {
+ModelService.update({}, {
+  
modelDescData:VdmUtil.filterNullValInObj($scope.state.modelSchema),
+  modelName: $routeParams.modelName,
+  project: $scope.state.project
+}, function (request) {
 if (request.successful) {
 $scope.state.modelSchema = request.modelSchema;
 SweetAlert.swal('', 'Updated the model 
successfully.', 'success');
@@ -216,7 +220,10 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, 
$routeParams, $locati
 loadingRequest.hide();
 });
 } else {
-ModelService.save({}, 
{modelDescData:$scope.state.modelSchema, project: $scope.state.project}, 
function (request) {
+ModelService.save({}, {
+  
modelDescData:VdmUtil.filterNullValInObj($scope.state.modelSchema),
+  project: $scope.state.project
+}, function (request) {
 if(request.successful) {
 
 

[10/27] kylin git commit: measure encoding length bug

[liyang]

measure encoding length bug

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a5295ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a5295ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a5295ba

Branch: refs/heads/KYLIN-1875
Commit: 1a5295bad21fc18766bb3c7ba6eee59678d42185
Parents: 5822492
Author: luguosheng <550175...@qq.com>
Authored: Thu Dec 1 17:54:35 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 18:17:35 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 4 ++--
 webapp/app/js/controllers/cubeEdit.js   | 2 +-
 webapp/app/js/controllers/cubeMeasures.js   | 8 
 3 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index f2b36fe..8192419 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -47,7 +47,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 var _encoding = item.encoding;
 var _valueLength ;
 var baseKey=item.encoding.replace(/:\d+/,'');
-if(needLengthKeyList.indexOf(baseKey)>=-1){
+if(needLengthKeyList.indexOf(baseKey)!=-1){
   var result=/:(\d+)/.exec(item.encoding);
   _valueLength=result?result[1]:0;
 }
@@ -78,7 +78,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 var version=$scope.getTypeVersion(item.encoding);
 var encodingType=$scope.removeVersion(item.encoding);
 
-if(needLengthKeyList.indexOf(encodingType)>=-1){
+if(needLengthKeyList.indexOf(encodingType)!=-1){
   encoding = encodingType+":"+item.valueLength;
 }else{
   encoding = encodingType;

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 0704a62..85bd4b1 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -78,7 +78,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 $scope.store.supportedEncoding = $scope.cubeConfig.encodings;
   })
   $scope.createFilter=function(type){
- if(type.indexOf("varchar")<=0){
+ if(type.indexOf("varchar")==-1){
return ['fixed_length_hex'];
  }else if(type!="date"){
return ['date'];

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 085338f..18f53c2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -45,7 +45,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 }
   }
   $scope.createFilter=function(type){
-if(type.indexOf("varchar")<=0){
+if(type.indexOf("varchar")==-1){
   return ['fixed_length_hex'];
 }else if(type!="date"){
   return ['date'];
@@ -60,7 +60,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 var encodings =$scope.store.supportedEncoding,filterEncoding=[];
 var filerList=$scope.createFilter(type);
 if($scope.isEdit) {
-  if (name && $scope.newMeasure.function.configuration) {
+  if (name && 
$scope.newMeasure.function.configuration&&$scope.newMeasure.function.configuration['topn.encoding.'
 + name]) {
 var version = 
$scope.newMeasure.function.configuration['topn.encoding_version.' + name] || 1;
 filterEncoding = VdmUtil.getFilterObjectListByOrFilterVal(encodings, 
'value', $scope.newMeasure.function.configuration['topn.encoding.' + 
name].replace(/:\d+/, "") + (version ? "[v" + version + "]" : "[v1]"), 
'suggest', true);
   }else{
@@ -119,7 +119,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 var 
version=$scope.newMeasure.function.configuration['topn.encoding_version.'+_name]||1;
 item=$scope.removeVersion(item);
 var baseKey=item.replace(/:\d+/,'');
-if(needLengthKeyList.indexOf(baseKey)>=-1){
+if(needLengthKeyList.indexOf(baseKey)!=-1){
   var result=/:(\d+)/.exec(item);
   

[26/27] kylin git commit: KYLIN-1875 fix PK-FK derive from each other

[liyang]

KYLIN-1875 fix PK-FK derive from each other


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8b955aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8b955aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8b955aa

Branch: refs/heads/KYLIN-1875
Commit: f8b955aac7458db7335b92c615871a15cdbadecb
Parents: e1d6f7f
Author: Yang Li 
Authored: Thu Dec 1 21:42:52 2016 +0800
Committer: Yang Li 
Committed: Fri Dec 2 21:02:01 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  8 +--
 .../org/apache/kylin/cube/model/CubeDesc.java   | 52 +++-
 .../storage/gtrecord/CubeTupleConverter.java|  2 +-
 .../gtrecord/GTCubeStorageQueryBase.java|  2 +-
 .../translate/DerivedFilterTranslator.java  |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  2 +-
 .../query/enumerator/LookupTableEnumerator.java |  2 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  2 +-
 .../hbase/cube/v1/CubeTupleConverter.java   |  2 +-
 9 files changed, 41 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4ba29af..d08aff2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -45,7 +45,6 @@ import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -55,6 +54,7 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TableDesc;
@@ -656,10 +656,10 @@ public class CubeManager implements IRealizationProvider {
 cubeMap.removeLocal(cubeName);
 }
 
-public LookupStringTable getLookupTable(CubeSegment cubeSegment, 
DimensionDesc dim) {
+public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc 
join) {
 
-String tableName = dim.getTableRef().getTableIdentity();
-String[] pkCols = dim.getJoin().getPrimaryKey();
+String tableName = join.getPKSide().getTableIdentity();
+String[] pkCols = join.getPrimaryKey();
 String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
 if (snapshotResPath == null)
 throw new IllegalStateException("No snaphot for table '" + 
tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() 
+ "/" + cubeSegment);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8b955aa/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f95cceb..f8c316c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -101,20 +102,20 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public static class DeriveInfo {
 public DeriveType type;
-public DimensionDesc dimension;
+public JoinDesc join;
 public TblColRef[] columns;
 public boolean isOneToOne; // only used when ref from derived to host
 
-DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] 
columns, boolean isOneToOne) {
+DeriveInfo(DeriveType type, JoinDesc join, 

[08/27] kylin git commit: KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally

[liyang]

 KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28ba1eae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28ba1eae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28ba1eae

Branch: refs/heads/KYLIN-1875
Commit: 28ba1eaeac8fd29ff3872f6d14604813a89b3a6e
Parents: 93bf0d0
Author: Hongbin Ma 
Authored: Thu Dec 1 11:35:34 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 11:46:00 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java   | 12 
 .../main/java/org/apache/kylin/cube/model/CubeDesc.java |  6 +-
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28ba1eae/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 766c04d..3c10826 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -79,7 +79,7 @@ abstract public class KylinConfigBase implements Serializable 
{
 protected KylinConfigBase(Properties props, boolean force) {
 this.properties = force ? props : BCC.check(props);
 }
-
+
 final protected String getOptional(String prop) {
 return getOptional(prop, null);
 }
@@ -161,7 +161,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public boolean isDevEnv() {
 return "DEV".equals(getOptional("kylin.env", "DEV"));
 }
-
+
 public String getDeployEnv() {
 return getOptional("kylin.env", "DEV");
 }
@@ -211,7 +211,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public Map getCubeCustomMeasureTypes() {
 return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
 }
-
+
 // 

 // DICTIONARY & SNAPSHOT
 // 

@@ -278,6 +278,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
 }
 
+public boolean isIgnoreCubeSignatureInconsistency() {
+return 
Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", 
"false"));
+}
+
 @Deprecated
 public int getCubeAggrGroupMaxSize() {
 return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", 
"12"));
@@ -298,7 +302,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public void setMaxBuildingSegments(int maxBuildingSegments) {
 setProperty("kylin.cube.max-building-segments", 
String.valueOf(maxBuildingSegments));
 }
-
+
 // 

 // JOB
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/28ba1eae/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 47063d0..327ce57 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -466,6 +466,11 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
  * @return
  */
 public boolean checkSignature() {
+if (this.getConfig().isIgnoreCubeSignatureInconsistency()) {
+logger.info("Skip checking cube signature");
+return true;
+}
+
 if (KylinVersion.getCurrentVersion().isCompatibleWith(new 
KylinVersion(getVersion())) && 
!KylinVersion.getCurrentVersion().isSignatureCompatibleWith(new 
KylinVersion(getVersion( {
 logger.info("checkSignature on {} is skipped as the its version is 
{} (not signature compatible but compatible) ", getName(), getVersion());
 return true;
@@ -1102,7 +1107,6 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 return newCubeDesc;
 }
 
-
 private Collection ensureOrder(Collection c) {
 TreeSet set = new TreeSet();
 for (Object o : c)

[17/27] kylin git commit: KYLIN-2192 More Robust Global Dictionary

[liyang]

KYLIN-2192 More Robust Global Dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a0ee798
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a0ee798
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a0ee798

Branch: refs/heads/KYLIN-1875
Commit: 4a0ee7989d5f8272592b980fce3f5716ca40d4c1
Parents: e562aaf
Author: sunyerui 
Authored: Mon Nov 21 21:26:34 2016 +0800
Committer: gaodayue 
Committed: Fri Dec 2 13:33:59 2016 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   |  12 +-
 .../apache/kylin/common/KylinConfigBase.java|   8 +
 .../apache/kylin/common/util/Dictionary.java|   2 +-
 .../model/validation/rule/DictionaryRule.java   |  78 +-
 .../validation/rule/DictionaryRuleTest.java |  28 +-
 .../apache/kylin/dict/AppendTrieDictionary.java | 285 +--
 .../kylin/dict/AppendTrieDictionaryChecker.java | 102 +++
 .../org/apache/kylin/dict/CachedTreeMap.java| 260 +++---
 .../kylin/dict/GlobalDictionaryBuilder.java |  36 +-
 .../kylin/dict/AppendTrieDictionaryTest.java| 150 +++-
 .../apache/kylin/dict/CachedTreeMapTest.java| 320 +---
 .../kylin/measure/bitmap/BitmapCounterTest.java |   6 +-
 ...t_kylin_cube_without_slr_left_join_desc.json |  16 +-
 .../localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv  | 804 +--
 .../flatten_data_for_without_slr_left_join.csv  | 804 +--
 .../test_kylin_inner_join_model_desc.json   |   3 +-
 .../test_kylin_inner_join_view_model_desc.json  |   3 +-
 .../test_kylin_left_join_model_desc.json|   3 +-
 .../test_kylin_left_join_view_model_desc.json   |   3 +-
 .../table/DEFAULT.TEST_KYLIN_FACT.json  |   8 +-
 .../source/hive/ITHiveTableReaderTest.java  |   2 +-
 .../query/sql_distinct_precisely/query00.sql|   2 +-
 .../query/sql_distinct_precisely/query01.sql|   2 +-
 .../query/sql_distinct_precisely/query02.sql|   2 +-
 .../query/sql_distinct_precisely/query03.sql|   3 +-
 .../query/sql_distinct_precisely/query04.sql|   3 +-
 .../query/sql_distinct_precisely/query05.sql|  25 -
 .../query/sql_distinct_precisely/query06.sql|  26 -
 .../query/sql_distinct_precisely/query07.sql|  24 -
 29 files changed, 1737 insertions(+), 1283 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 8068fd1..677b713 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -403,13 +403,13 @@ public class FactTableGenerator {
 }
 
 private String createRandomCell(ColumnDesc cDesc) {
-String type = cDesc.getTypeName();
-String s = type.toLowerCase();
-if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
+DataType type =cDesc.getType();
+String s = type.getName();
+if (s.equals("char") || s.equals("varchar")) {
 StringBuilder sb = new StringBuilder();
-for (int i = 0; i < 2; i++) {
-sb.append((char) ('a' + r.nextInt(10)));// there are 10*10
-// possible strings
+int len = Math.min(type.getPrecision(), 3);
+for (int i = 0; i < len; i++) {
+sb.append((char) ('a' + r.nextInt(10)));  // cardinality at 
most 10x10x10
 }
 return sb.toString();
 } else if (s.equals("bigint") || s.equals("int") || 
s.equals("tinyint") || s.equals("smallint")) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3c10826..f46c185 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -232,6 +232,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "1000"));
 }
 
+public int getAppendDictMaxVersions() {
+return 
Integer.parseInt(getOptional("kylin.dictionary.append-max-versions", "3"));
+}
+
+public int getAppendDictVersionTTL() {
+return 

[02/27] kylin git commit: Revert "KYLIN-2195 could run get-properties.sh in source code dir"

[liyang]

Revert "KYLIN-2195 could run get-properties.sh in source code dir"

This reverts commit 8506e4d4f75fbbda518997cac97e229fa5a0.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3186d176
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3186d176
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3186d176

Branch: refs/heads/KYLIN-1875
Commit: 3186d176113596c71a47d98a927564cefff3b123
Parents: 8506e4d
Author: lidongsjtu 
Authored: Tue Nov 29 19:11:50 2016 +0800
Committer: lidongsjtu 
Committed: Tue Nov 29 19:11:50 2016 +0800

--
 build/bin/get-properties.sh | 12 ++--
 1 file changed, 2 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3186d176/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 0fa4f6f..170442d 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -19,21 +19,13 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-
 if [ $# != 1 ]
 then
 echo 'invalid input'
 exit -1
 fi
 
-if [[ $CI_MODE == 'true' ]]; then
-cd $dir
-job_jar=$(ls ../../assembly/target/kylin-*-job.jar)
-tool_jar=$(ls ../../tool/target/kylin-tool-*.jar|grep -v assembly)
-else
-job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
-tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
-fi
-
+job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
+tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
 result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 
2>/dev/null`
 echo "$result"

[27/27] kylin git commit: minor, rollback ITKylinQueryTest

[liyang]

minor, rollback ITKylinQueryTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/84ce3ad6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/84ce3ad6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/84ce3ad6

Branch: refs/heads/KYLIN-1875
Commit: 84ce3ad62ddde9e174c6f9ffadb85cc934b66d08
Parents: f8b955a
Author: Yang Li 
Authored: Fri Dec 2 06:32:25 2016 +0800
Committer: Yang Li 
Committed: Fri Dec 2 21:02:01 2016 +0800

--
 .../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/84ce3ad6/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 6d91753..90324b5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -48,7 +48,7 @@ import org.junit.rules.ExpectedException;
 
 import com.google.common.collect.Maps;
 
-//@Ignore("KylinQueryTest is contained by ITCombinationTest")
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
 public class ITKylinQueryTest extends KylinTestBase {
 
 @Rule

[21/27] kylin git commit: KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization

[liyang]

KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb686a94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb686a94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb686a94

Branch: refs/heads/KYLIN-1875
Commit: eb686a9428d930a60ec87883a2c253fb07f80a72
Parents: 0fd1ed6
Author: Li Yang 
Authored: Fri Dec 2 18:34:29 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 18:34:29 2016 +0800

--
 .../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb686a94/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 1e836b7..f6e687b 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -151,17 +151,21 @@ public class BuiltInFunctionTupleFilter extends 
FunctionTupleFilter {
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 BytesUtil.writeUTFString(name, buffer);
+buffer.put((byte) (isReversed ? 1 : 0));
 }
 
 @Override
 public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 this.name = BytesUtil.readUTFString(buffer);
+this.isReversed = buffer.get() != 0;
 this.initMethod();
 }
 
 @Override
 public String toString() {
 StringBuilder sb = new StringBuilder();
+if (isReversed)
+sb.append("NOT ");
 sb.append(name);
 sb.append("(");
 for (int i = 0; i < methodParams.size(); i++) {

[20/27] kylin git commit: KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result

[liyang]

KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fd1ed6f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fd1ed6f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fd1ed6f

Branch: refs/heads/KYLIN-1875
Commit: 0fd1ed6f14758bb38f1abf7a7ed9e24ed818f50f
Parents: 0eebd7d
Author: zhengdong 
Authored: Wed Nov 23 16:13:59 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 18:32:24 2016 +0800

--
 .../java/org/apache/kylin/gridtable/GTUtil.java | 14 
 .../metadata/filter/CompareTupleFilter.java |  2 ++
 .../metadata/filter/TupleFilterSerializer.java  |  1 +
 .../resources/query/sql_derived/query12.sql | 24 
 .../kylin/query/relnode/OLAPFilterRel.java  |  3 +++
 .../common/coprocessor/FilterDecorator.java | 14 
 6 files changed, 58 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
old mode 100644
new mode 100755
index a4e574c..7496778
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -196,6 +196,20 @@ public class GTUtil {
 result = newCompareFilter;
 }
 break;
+case NOTIN:
+Set notInValues = Sets.newHashSet();
+for (Object value : constValues) {
+code = translate(col, value, 0);
+if (code != null)
+notInValues.add(code);
+}
+if (notInValues.isEmpty()) {
+result = ConstantTupleFilter.TRUE;
+} else {
+newCompareFilter.addChild(new 
ConstantTupleFilter(notInValues));
+result = newCompareFilter;
+}
+break;
 case NEQ:
 code = translate(col, firstValue, 0);
 if (code == null) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
old mode 100644
new mode 100755
index c7a3721..f2af735
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -59,6 +59,8 @@ public class CompareTupleFilter extends TupleFilter {
 private CompareTupleFilter(CompareTupleFilter another) {
 super(new ArrayList(another.children), another.operator);
 this.column = another.column;
+this.firstCondValue = another.getFirstValue();
+this.function = another.getFunction();
 this.conditionValues = new HashSet();
 this.conditionValues.addAll(another.conditionValues);
 this.dynamicVariables = new HashMap();

http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
old mode 100644
new mode 100755
index 2df474e..63153ef
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -170,6 +170,7 @@ public class TupleFilterSerializer {
 case GT:
 case GTE:
 case IN:
+case NOTIN:
 case ISNULL:
 case ISNOTNULL:
 filter = new CompareTupleFilter(op);

http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/kylin-it/src/test/resources/query/sql_derived/query12.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_derived/query12.sql 
b/kylin-it/src/test/resources/query/sql_derived/query12.sql
new file mode 100755
index 000..959a59c
--- 

[16/27] kylin git commit: KYLIN-2192 More Robust Global Dictionary

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java 
b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
index df64f3f..381e0b1 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
@@ -17,11 +17,12 @@
 */
 package org.apache.kylin.dict;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.*;
@@ -91,39 +92,26 @@ public class CachedTreeMapTest {
 public static class CachedFileFilter implements FileFilter {
 @Override
 public boolean accept(File pathname) {
-return pathname.getName().startsWith("cached_");
+return pathname.getName().startsWith(CachedTreeMap.CACHED_PREFIX);
 }
 }
 
-public static final String baseDir = "/tmp/kylin_cachedtreemap_test/";
-public static final String backupDir = 
"/tmp/kylin_cachedtreemap_test.bak/";
-public static final String tmpDir = "/tmp/kylin_cachedtreemap_test.tmp/";
-
-private static void cleanup() {
-File dir = new File(baseDir);
-if (dir.exists()) {
-for (File f : dir.listFiles()) {
-f.delete();
-}
-dir.delete();
-}
-
-dir = new File(tmpDir);
-if (dir.exists()) {
-for (File f : dir.listFiles()) {
-f.delete();
-}
-dir.delete();
+public static class VersionFilter implements FileFilter {
+@Override
+public boolean accept(File pathname) {
+return pathname.getName().startsWith(CachedTreeMap.VERSION_PREFIX);
 }
+}
 
-dir = new File(backupDir);
-if (dir.exists()) {
-for (File f : dir.listFiles()) {
-f.delete();
-}
-dir.delete();
-}
+public static final String baseDir = "/tmp/kylin_cachedtreemap_test/";
+public static final String workingDir = 
"/tmp/kylin_cachedtreemap_test/working";
 
+private static void cleanup() {
+Configuration conf = new Configuration();
+Path basePath = new Path(baseDir);
+try {
+FileSystem.get(basePath.toUri(), conf).delete(basePath, true);
+} catch (IOException e) {}
 VALUE_WRITE_ERROR_TOGGLE = false;
 }
 
@@ -139,154 +127,240 @@ public class CachedTreeMapTest {
 
 @Test
 public void testCachedTreeMap() throws IOException {
-CachedTreeMap map = 
CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
-
.persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
+CachedTreeMap map = createMutableMap();
 map.put(Key.of(1), Value.of("a"));
 map.put(Key.of(2), Value.of("b"));
 map.put(Key.of(3), Value.of("c"));
 map.put(Key.of(4), Value.of("d"));
 map.put(Key.of(5), Value.of("e"));
 
-File dir = new File(tmpDir);
+File dir = new File(workingDir);
 assertEquals(3, dir.listFiles(new CachedFileFilter()).length);
 
-DataOutputStream out = new DataOutputStream(new 
FileOutputStream(tmpDir+"/.index"));
-map.write(out);
-out.flush();
-out.close();
-map.commit(false);
+flushAndCommit(map, true, true, false);
+assertFalse(new File(workingDir).exists());
 
-dir = new File(baseDir);
+dir = new File(map.getLatestVersion());
 assertEquals(5, dir.listFiles(new CachedFileFilter()).length);
 
-DataInputStream in = new DataInputStream(new 
FileInputStream(baseDir+".index"));
-CachedTreeMap map2 = 
CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
-
.persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
-map2.readFields(in);
+CachedTreeMap map2 = createImmutableMap();
 assertEquals(5, map2.size());
 assertEquals("b", ((Value)map2.get(Key.of(2))).valueStr);
 
 try {
 map2.put(Key.of(6), Value.of("f"));
 fail("Should be error when put value into immutable map");
-} catch (AssertionError error) {
+} catch (AssertionError error) {}
+}
+
+@Test
+public void testMultiVersions() throws IOException, InterruptedException {
+CachedTreeMap map = createMutableMap();
+Thread.sleep(3000);
+map.put(Key.of(1), Value.of("a"));
+map.put(Key.of(2), 

[25/27] kylin git commit: KYLIN-1875 Support measure on non-root-fact-table

[liyang]

KYLIN-1875 Support measure on non-root-fact-table


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1d6f7fe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1d6f7fe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1d6f7fe

Branch: refs/heads/KYLIN-1875
Commit: e1d6f7fed236ab1458a8663f0823eebfc89f6882
Parents: 47e922e
Author: Li Yang 
Authored: Thu Dec 1 18:59:08 2016 +0800
Committer: Yang Li 
Committed: Fri Dec 2 21:01:59 2016 +0800

--
 .../kylin/metadata/project/ProjectL2Cache.java  | 26 +---
 .../template/cube_desc/kylin_sales_cube.json| 18 +++---
 .../kylin/query/relnode/OLAPTableScan.java  |  4 +++
 3 files changed, 30 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d6f7fe/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 82c0de3..14d7843 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
@@ -126,17 +127,17 @@ class ProjectL2Cache {
 return Collections.unmodifiableSet(tableCache.realizations);
 }
 
-public List listEffectiveRewriteMeasures(String project, 
String factTable, boolean onlyRewriteMeasure) {
-Set realizations = getRealizationsByTable(project, 
factTable);
+public List listEffectiveRewriteMeasures(String project, 
String table, boolean onlyRewriteMeasure) {
+Set realizations = getRealizationsByTable(project, 
table);
 List result = Lists.newArrayList();
 for (IRealization r : realizations) {
-if (r.getModel().isFactTable(factTable) && r.isReady()) {
-for (MeasureDesc m : r.getMeasures()) {
-FunctionDesc func = m.getFunction();
-if (onlyRewriteMeasure) {
-if (func.needRewrite())
-result.add(m);
-} else {
+if (!r.isReady())
+continue;
+
+for (MeasureDesc m : r.getMeasures()) {
+FunctionDesc func = m.getFunction();
+if (belongToTable(func, table, r.getModel())) {
+if (!onlyRewriteMeasure || func.needRewrite()) {
 result.add(m);
 }
 }
@@ -145,6 +146,13 @@ class ProjectL2Cache {
 return result;
 }
 
+private boolean belongToTable(FunctionDesc func, String table, 
DataModelDesc model) {
+// measure belong to the first column parameter's table
+List cols = func.getParameter().getColRefs();
+String belongTo = cols.isEmpty() ? 
model.getRootFactTable().getTableIdentity() : cols.get(0).getTable();
+return belongTo.equals(table);
+}
+
 // 

 // build the cache
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d6f7fe/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 48eef46..2a27305 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -103,26 +103,26 @@
   "returntype" : "decimal(19,4)"
 }
   }, {
-"name" : "GMV_MIN",
+"name" : "BUYER_LEVEL_SUM",
 "function" : {
-  "expression" : "MIN",
+  "expression" : "SUM",
   "parameter" : {
 "type" : "column",
-"value" : "PRICE",
+"value" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
 "next_parameter" : null
   },
-  "returntype" : "decimal(19,4)"
+  "returntype" : "bigint"
 }
   }, {
-"name" : "GMV_MAX",
+"name" : "SELLER_LEVEL_SUM",
 "function" : {
-  "expression" : "MAX",
+  

[23/27] kylin git commit: KYLIN-1875 fix ParameterDesc equals() & hashcode()

[liyang]

KYLIN-1875 fix ParameterDesc equals() & hashcode()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6a1b232
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6a1b232
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6a1b232

Branch: refs/heads/KYLIN-1875
Commit: e6a1b232d9e12eae10978d1fa555d2214c90925c
Parents: a2a36be
Author: Yang Li 
Authored: Thu Dec 1 06:58:08 2016 +0800
Committer: Yang Li 
Committed: Fri Dec 2 20:59:49 2016 +0800

--
 .../kylin/metadata/model/FunctionDesc.java  |  1 +
 .../kylin/metadata/model/ParameterDesc.java | 36 +++-
 2 files changed, 28 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a1b232/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index dfa6f3b..b9e5543 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -248,6 +248,7 @@ public class FunctionDesc {
 final int prime = 31;
 int result = 1;
 result = prime * result + ((expression == null) ? 0 : 
expression.hashCode());
+result = prime * result + ((returnType == null) ? 0 : 
returnType.hashCode());
 result = prime * result + ((isCount() || parameter == null) ? 0 : 
parameter.hashCode());
 return result;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a1b232/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 329799f..4a95fea 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -22,8 +22,10 @@ import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableList;
 
 /**
  */
@@ -36,9 +38,10 @@ public class ParameterDesc {
 private String value;
 
 @JsonProperty("next_parameter")
+@JsonInclude(JsonInclude.Include.NON_NULL)
 private ParameterDesc nextParameter;
 
-private List colRefs;
+private List colRefs = ImmutableList.of();
 
 public String getType() {
 return type;
@@ -89,21 +92,36 @@ public class ParameterDesc {
 
 ParameterDesc that = (ParameterDesc) o;
 
-if (nextParameter != null ? !nextParameter.equals(that.nextParameter) 
: that.nextParameter != null)
-return false;
 if (type != null ? !type.equals(that.type) : that.type != null)
 return false;
-if (value != null ? !value.equals(that.value) : that.value != null)
-return false;
-
-return true;
+
+ParameterDesc p = this, q = that;
+int refi = 0, refj = 0;
+for (; p != null && q != null; p = p.nextParameter, q = 
q.nextParameter) {
+if (p.isColumnType()) {
+if (q.isColumnType() == false)
+return false;
+if (refi >= this.colRefs.size() || refj >= that.colRefs.size())
+return false;
+if (this.colRefs.get(refi).equals(that.colRefs.get(refj)) == 
false)
+return false;
+refi++;
+refj++;
+} else {
+if (q.isColumnType() == true)
+return false;
+if (p.value.equals(q.value) == false)
+return false;
+}
+}
+
+return p == null && q == null;
 }
 
 @Override
 public int hashCode() {
 int result = type != null ? type.hashCode() : 0;
-result = 31 * result + (value != null ? value.hashCode() : 0);
-result = 31 * result + (nextParameter != null ? 
nextParameter.hashCode() : 0);
+result = 31 * result + (colRefs != null ? colRefs.hashCode() : 0);
 return result;
 }
 

[06/27] kylin git commit: KYLIN-2217 Code review, refactor IDictionaryBuilder

[liyang]

KYLIN-2217 Code review, refactor IDictionaryBuilder


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30cb1ac6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30cb1ac6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30cb1ac6

Branch: refs/heads/KYLIN-1875
Commit: 30cb1ac678b063ac164acd27591e8a1d0becafa0
Parents: 1af08e4
Author: Li Yang 
Authored: Wed Nov 30 15:30:15 2016 +0800
Committer: Yang Li 
Committed: Wed Nov 30 21:00:17 2016 +0800

--
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  |   2 +-
 .../apache/kylin/dict/DictionaryGenerator.java  | 165 ---
 .../apache/kylin/dict/DictionaryManager.java|  16 +-
 .../apache/kylin/dict/DictionaryProvider.java   |   4 +-
 .../dict/DictionaryReducerLocalGenerator.java   | 156 --
 .../kylin/dict/GlobalDictionaryBuilder.java |  36 ++--
 .../apache/kylin/dict/IDictionaryBuilder.java   |  13 +-
 .../dict/IDictionaryReducerLocalBuilder.java|  31 
 .../kylin/dict/DictionaryProviderTest.java  |  63 +++
 .../engine/mr/steps/CreateDictionaryJob.java|  42 ++---
 .../mr/steps/FactDistinctColumnsReducer.java| 115 +
 .../mr/steps/FactDistinctHiveColumnsMapper.java |   9 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   6 +-
 13 files changed, 231 insertions(+), 427 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index a4e1df0..163c6ca 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -60,7 +60,7 @@ public class DictionaryGeneratorCLI {
 for (TblColRef col : 
cubeSeg.getCubeDesc().getAllColumnsNeedDictionaryBuilt()) {
 logger.info("Building dictionary for " + col);
 ReadableTable inpTable = decideInputTable(cubeSeg.getModel(), col, 
factTableValueProvider);
-if (config.isReducerLocalBuildDict() && dictProvider != null) {
+if (dictProvider != null) {
 Dictionary dict = dictProvider.getDictionary(col);
 if (dict != null) {
 cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);

http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 810a392..cd13d59 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -19,12 +19,11 @@
 package org.apache.kylin.dict;
 
 import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.slf4j.Logger;
@@ -40,9 +39,7 @@ public class DictionaryGenerator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryGenerator.class);
 
-private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
-
-public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
+public static IDictionaryBuilder newDictionaryBuilder(DataType dataType) {
 Preconditions.checkNotNull(dataType, "dataType cannot be null");
 
 // build dict, case by data type
@@ -57,16 +54,33 @@ public class DictionaryGenerator {
 } else {
 builder = new StringDictBuilder();
 }
+return builder;
+}
 
-return buildDictionary(builder, null, valueEnumerator);
+public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
+return buildDictionary(newDictionaryBuilder(dataType), null, 
valueEnumerator);
 }
 
-public static Dictionary buildDictionary(IDictionaryBuilder 
builder, DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator) 
throws IOException {
+static 

[11/27] kylin git commit: KYLIN-2180 minor, get owenr project in CubeDesc.init()

[liyang]

KYLIN-2180 minor, get owenr project in CubeDesc.init()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9fc8f5e2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9fc8f5e2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9fc8f5e2

Branch: refs/heads/KYLIN-1875
Commit: 9fc8f5e207b5111fffaec3335efe0c8947353cfa
Parents: 1a5295b
Author: Li Yang 
Authored: Fri Dec 2 11:39:34 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 11:39:34 2016 +0800

--
 .../main/java/org/apache/kylin/cube/model/CubeDesc.java  | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9fc8f5e2/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 327ce57..853571c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -64,6 +64,9 @@ import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -519,11 +522,16 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public void init(KylinConfig config) {
 this.errors.clear();
-this.config = KylinConfigExt.createInstance(config, 
overrideKylinProps);
 
 checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank");
 checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc(%s) has 
blank modelName", name);
 
+// note CubeDesc.name == CubeInstance.name
+List ownerPrj = 
ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name);
+logger.info("CubeDesc '" + name + "' is owned by " + ownerPrj);
+
+this.config = KylinConfigExt.createInstance(config, 
overrideKylinProps);
+
 this.model = 
MetadataManager.getInstance(config).getDataModelDesc(modelName);
 checkNotNull(this.model, "DateModelDesc(%s) not found", modelName);
 
@@ -823,6 +831,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 return col;
 }
 
+@SuppressWarnings("deprecation")
 private void initMeasureColumns() {
 if (measures == null || measures.isEmpty()) {
 return;

[14/27] kylin git commit: KYLIN-2192 More Robust Global Dictionary

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
index a3948c3..0c163a6 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
@@ -19,6 +19,6 @@
 select lstg_format_name, cal_dt,
  sum(price) as GMV,
  count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
+ count(distinct user_id) as user_count
  from test_kylin_fact
  group by lstg_format_name, cal_dt

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
index e8579ef..62142a2 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
@@ -19,7 +19,7 @@
 select lstg_format_name,
  sum(price) as GMV,
  count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
+ count(distinct user_id) as user_count
  from test_kylin_fact
  where lstg_format_name='FP-GTC'
  group by lstg_format_name

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
index 48f49e9..88ce532 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
@@ -19,7 +19,7 @@
 select lstg_format_name,
  sum(price) as GMV,
  count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
+ count(distinct user_id) as user_count
  from test_kylin_fact
  where lstg_format_name='FP-GTC'
  group by lstg_format_name

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
index dbc2fac..7f739e7 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
@@ -17,7 +17,8 @@
 --
 
 select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count
+ , count(1) as TRANS_CNT
+ , count(distinct user_id) as user_count
  , count(distinct site_name) as site_count
  from test_kylin_fact
  inner JOIN edw.test_cal_dt as test_cal_dt

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
index 69006ce..13de5a7 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
@@ -17,7 +17,8 @@
 --
 
 select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count
+ , count(1) as TRANS_CNT
+ , count(distinct user_id) as user_count
  , count(distinct site_name) as site_count
  from test_kylin_fact
  inner JOIN edw.test_cal_dt as test_cal_dt

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql 
b/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
deleted file mode 100644
index dea09f7..000
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
+++ /dev/null
@@ -1,25 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with 

[1/2] kylin git commit: KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 0eebd7d8a -> eb686a942


KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fd1ed6f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fd1ed6f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fd1ed6f

Branch: refs/heads/master
Commit: 0fd1ed6f14758bb38f1abf7a7ed9e24ed818f50f
Parents: 0eebd7d
Author: zhengdong 
Authored: Wed Nov 23 16:13:59 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 18:32:24 2016 +0800

--
 .../java/org/apache/kylin/gridtable/GTUtil.java | 14 
 .../metadata/filter/CompareTupleFilter.java |  2 ++
 .../metadata/filter/TupleFilterSerializer.java  |  1 +
 .../resources/query/sql_derived/query12.sql | 24 
 .../kylin/query/relnode/OLAPFilterRel.java  |  3 +++
 .../common/coprocessor/FilterDecorator.java | 14 
 6 files changed, 58 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
old mode 100644
new mode 100755
index a4e574c..7496778
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -196,6 +196,20 @@ public class GTUtil {
 result = newCompareFilter;
 }
 break;
+case NOTIN:
+Set notInValues = Sets.newHashSet();
+for (Object value : constValues) {
+code = translate(col, value, 0);
+if (code != null)
+notInValues.add(code);
+}
+if (notInValues.isEmpty()) {
+result = ConstantTupleFilter.TRUE;
+} else {
+newCompareFilter.addChild(new 
ConstantTupleFilter(notInValues));
+result = newCompareFilter;
+}
+break;
 case NEQ:
 code = translate(col, firstValue, 0);
 if (code == null) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
old mode 100644
new mode 100755
index c7a3721..f2af735
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -59,6 +59,8 @@ public class CompareTupleFilter extends TupleFilter {
 private CompareTupleFilter(CompareTupleFilter another) {
 super(new ArrayList(another.children), another.operator);
 this.column = another.column;
+this.firstCondValue = another.getFirstValue();
+this.function = another.getFunction();
 this.conditionValues = new HashSet();
 this.conditionValues.addAll(another.conditionValues);
 this.dynamicVariables = new HashMap();

http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
old mode 100644
new mode 100755
index 2df474e..63153ef
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -170,6 +170,7 @@ public class TupleFilterSerializer {
 case GT:
 case GTE:
 case IN:
+case NOTIN:
 case ISNULL:
 case ISNOTNULL:
 filter = new CompareTupleFilter(op);

http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/kylin-it/src/test/resources/query/sql_derived/query12.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_derived/query12.sql 

[2/2] kylin git commit: KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization

[liyang]

KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb686a94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb686a94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb686a94

Branch: refs/heads/master
Commit: eb686a9428d930a60ec87883a2c253fb07f80a72
Parents: 0fd1ed6
Author: Li Yang 
Authored: Fri Dec 2 18:34:29 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 18:34:29 2016 +0800

--
 .../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb686a94/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 1e836b7..f6e687b 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -151,17 +151,21 @@ public class BuiltInFunctionTupleFilter extends 
FunctionTupleFilter {
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 BytesUtil.writeUTFString(name, buffer);
+buffer.put((byte) (isReversed ? 1 : 0));
 }
 
 @Override
 public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 this.name = BytesUtil.readUTFString(buffer);
+this.isReversed = buffer.get() != 0;
 this.initMethod();
 }
 
 @Override
 public String toString() {
 StringBuilder sb = new StringBuilder();
+if (isReversed)
+sb.append("NOT ");
 sb.append(name);
 sb.append("(");
 for (int i = 0; i < methodParams.size(); i++) {

[2/2] kylin git commit: KYLIN-2245 slim ISegment a bit

[liyang]

KYLIN-2245 slim ISegment a bit


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0eebd7d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0eebd7d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0eebd7d8

Branch: refs/heads/master
Commit: 0eebd7d8ab10b6b2b4823d6a23eaa774ac36989e
Parents: 3ca5395
Author: Li Yang 
Authored: Fri Dec 2 17:23:16 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 17:26:13 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeSegment.java | 20 +++-
 .../apache/kylin/metadata/model/ISegment.java   |  1 -
 .../apache/kylin/metadata/model/Segments.java   | 12 +++-
 3 files changed, 14 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 1fc28eb..5339d81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -38,6 +38,7 @@ import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.ISegment;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;
 
@@ -350,29 +351,14 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 this.sourceOffsetEnd = sourceOffsetEnd;
 }
 
-public boolean dateRangeOverlaps(CubeSegment seg) {
-return dateRangeStart < seg.dateRangeEnd && seg.dateRangeStart < 
dateRangeEnd;
-}
-
-public boolean dateRangeContains(CubeSegment seg) {
-return dateRangeStart <= seg.dateRangeStart && seg.dateRangeEnd <= 
dateRangeEnd;
-}
-
 // date range is used in place of source offsets when offsets are missing
 public boolean sourceOffsetOverlaps(CubeSegment seg) {
-if (isSourceOffsetsOn())
-return sourceOffsetStart < seg.sourceOffsetEnd && 
seg.sourceOffsetStart < sourceOffsetEnd;
-else
-return dateRangeOverlaps(seg);
+return Segments.sourceOffsetOverlaps(this, seg);
 }
 
 // date range is used in place of source offsets when offsets are missing
-@Override
 public boolean sourceOffsetContains(ISegment seg) {
-if (isSourceOffsetsOn())
-return sourceOffsetStart <= ((CubeSegment) seg).sourceOffsetStart 
&& ((CubeSegment) seg).sourceOffsetEnd <= sourceOffsetEnd;
-else
-return dateRangeContains(((CubeSegment) seg));
+return Segments.sourceOffsetContains(this, seg);
 }
 
 public void validate() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index e97f4f4..e3fcdcb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -36,5 +36,4 @@ public interface ISegment{
 
 public long getLastBuildTime();
 
-public boolean sourceOffsetContains(ISegment seg);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 104c2af..f0a58cb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -21,6 +21,16 @@ package org.apache.kylin.metadata.model;
 import java.util.ArrayList;
 
 public class Segments extends ArrayList {
+
+private static final long serialVersionUID = 1L;
+
+public static boolean sourceOffsetContains(ISegment a, ISegment b) {
+return a.getSourceOffsetStart() <= b.getSourceOffsetStart() && 
b.getSourceOffsetEnd() <= a.getSourceOffsetEnd();
+}
+
+public static boolean sourceOffsetOverlaps(ISegment a, ISegment b) {
+return a.getSourceOffsetStart() <