date:20161027

[2/2] kylin git commit: KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as QueryId

2016-10-27 Thread gaodayue

KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as 
QueryId


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46a894db
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46a894db
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46a894db

Branch: refs/heads/v1.6.0-rc1
Commit: 46a894db60e7538468d6e7459ea760bca286249e
Parents: 5d4d639
Author: gaodayue 
Authored: Thu Oct 27 12:14:49 2016 +0800
Committer: gaodayue 
Committed: Fri Oct 28 13:32:38 2016 +0800

--
 .../apache/kylin/rest/service/QueryService.java |   8 +-
 .../kylin/rest/util/QueryIdGenerator.java   |  47 
 .../kylin/rest/util/QueryIdGeneratorTest.java   | 108 ---
 .../coprocessor/endpoint/CubeVisitService.java  |   2 +-
 4 files changed, 4 insertions(+), 161 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/46a894db/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 94f2dd5..81af044 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -39,6 +39,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 
 import javax.annotation.PostConstruct;
 import javax.sql.DataSource;
@@ -72,7 +73,6 @@ import org.apache.kylin.rest.model.TableMeta;
 import org.apache.kylin.rest.request.PrepareSqlRequest;
 import org.apache.kylin.rest.request.SQLRequest;
 import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.kylin.rest.util.QueryIdGenerator;
 import org.apache.kylin.rest.util.QueryUtil;
 import org.apache.kylin.rest.util.Serializer;
 import org.apache.kylin.rest.util.TableauInterceptor;
@@ -117,8 +117,6 @@ public class QueryService extends BasicService {
 private final String hbaseUrl;
 private final String userTableName;
 
-private QueryIdGenerator queryIdGenerator = new QueryIdGenerator();
-
 @Autowired
 private CacheManager cacheManager;
 
@@ -325,7 +323,7 @@ public class QueryService extends BasicService {
 throw new InternalErrorException("Query is not allowed in " + 
serverMode + " mode.");
 }
 
-final String queryId = 
queryIdGenerator.nextId(sqlRequest.getProject());
+final String queryId = UUID.randomUUID().toString();
 
 Map toggles = new HashMap<>();
 toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
@@ -334,7 +332,7 @@ public class QueryService extends BasicService {
 }
 BackdoorToggles.setToggles(toggles);
 
-try (SetThreadName ignored = new SetThreadName("Query-%s", queryId)) {
+try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
 String sql = sqlRequest.getSql();
 String project = sqlRequest.getProject();
 logger.info("Using project: " + project);

http://git-wip-us.apache.org/repos/asf/kylin/blob/46a894db/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
deleted file mode 100644
index 2dd19c2..000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.util;
-
-import org.apache.commons.lang3.time.FastDateFormat;
-
-import javax.annotation.Nonnull;
-import javax.annotation.concurrent.ThreadSafe;
-import

svn commit: r1766917 - in /kylin/site: docs15/gettingstarted/faq.html feed.xml

2016-10-27 Thread lidong

Author: lidong
Date: Fri Oct 28 02:28:18 2016
New Revision: 1766917

URL: http://svn.apache.org/viewvc?rev=1766917=rev
Log:
update faq for EMR

Modified:
kylin/site/docs15/gettingstarted/faq.html
kylin/site/feed.xml

Modified: kylin/site/docs15/gettingstarted/faq.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs15/gettingstarted/faq.html?rev=1766917=1766916=1766917=diff
==
--- kylin/site/docs15/gettingstarted/faq.html (original)
+++ kylin/site/docs15/gettingstarted/faq.html Fri Oct 28 02:28:18 2016
@@ -2104,26 +2104,52 @@
1.
 âbin/find-hive-dependency.shâ can locate hive/hcat jars in local, but 
Kylin reports error like âjava.lang.NoClassDefFoundError: 
org/apache/hive/hcatalog/mapreduce/HCatInputFormatâ
 
 
-  
-Kylin need many dependent jars (hadoop/hive/hcat/hbase/kafka) on 
classpath to work, but Kylin doesnât ship them. It will seek these jars from 
your local machine by running commands like hbase classpath, hive -e set etc. The founded jarsâ path will 
be appended to the environment variable HBASE_CLASSPATH (Kylin uses 
hbase shell command to start up, which 
will read this). But in some Hadoop distribution (like EMR 5.0), the hbase shell doesnât keep the origin HBASE_CLASSPATH value, that causes the 
âNoClassDefFoundErrorâ.
-  
-  
-To fix this, find the hbase shell script (in hbase/bin folder), and 
search HBASE_CLASSPATH, check whether it overwrite the value like :
-  
+  Kylin need many dependent jars (hadoop/hive/hcat/hbase/kafka) on 
classpath to work, but Kylin doesnât ship them. It will seek these jars from 
your local machine by running commands like hbase classpath, hive -e set etc. The founded jarsâ path will 
be appended to the environment variable HBASE_CLASSPATH (Kylin uses 
hbase shell command to start up, which 
will read this). But in some Hadoop distribution (like EMR 5.0), the hbase shell doesnât keep the origin HBASE_CLASSPATH value, that causes the 
âNoClassDefFoundErrorâ.
 
 
-export 
HBASE_CLASSPATH=$HADOOP_CONF:$HADOOP_HOME/*:$HADOOP_HOME/lib/*:$ZOOKEEPER_HOME/*:$ZOOKEEPER_HOME/lib/*
+To fix this, find the hbase shell script (in hbase/bin or hbase/conf 
folder), and search HBASE_CLASSPATH, check whether it overwrite the 
value like :
+
+  export 
HBASE_CLASSPATH=$HADOOP_CONF:$HADOOP_HOME/*:$HADOOP_HOME/lib/*:$ZOOKEEPER_HOME/*:$ZOOKEEPER_HOME/lib/*
+
+
+
+If true, change it to keep the origin value like:
+
+  export 
HBASE_CLASSPATH=$HADOOP_CONF:$HADOOP_HOME/*:$HADOOP_HOME/lib/*:$ZOOKEEPER_HOME/*:$ZOOKEEPER_HOME/lib/*:$HBASE_CLASSPATH
+
+
 
 
-  If true, change it to keep the origin value like:
+  For EMR5.0, need change the hbase-env.sh
 
 
-export 
HBASE_CLASSPATH=$HADOOP_CONF:$HADOOP_HOME/*:$HADOOP_HOME/lib/*:$ZOOKEEPER_HOME/*:$ZOOKEEPER_HOME/lib/*:$HBASE_CLASSPATH
+ sudo vi 
/usr/lib/hbase/conf/hbase-env.sh
+
+
+
+in around line 30 it was :
+
+export 
HBASE_CLASSPATH=/etc/hadoop/conf
+
+
+
+change to :
+
+export 
HBASE_CLASSPATH=/etc/hadoop/conf:$HBASE_CLASSPATH
+
+
+
+then restart Kylin.
 
 2.
 Get âjava.lang.IllegalArgumentException: Too high cardinality is not 
suitable for dictionary â cardinality: 5220674â in âBuild Dimension 
Dictionaryâ step
 
 
-  Kylin uses âDictionaryâ encoding to encode/decode the dimension 
values (check this blog); 
Usually a dimensionâs cardinality is less than millions, so the âDictâ 
encoding is good to use. As dictionary need be persisted and loaded into 
memory, if a dimensionâs cardinality is very high, the memory footprint will 
be tremendous, so Kylin add a check on this. If you see this error, suggest to 
identify the UHC dimension first and then re-evaluate the design (whether need 
to make that as dimension?). If must keep it, you can by-pass this error with 
couple ways: 1) change to use other encoding (like fixed_length, integer) 2) or set a bigger value for kylin.dictionary.max.cardinality in conf/kylin.properties.
+  Kylin uses âDictionaryâ encoding to encode/decode the dimension 
values (check this blog); 
Usually a dimensionâs cardinality is less than millions, so the âDictâ 
encoding is good to use. As dictionary need be persisted and loaded into 
memory, if a dimensionâs cardinality is very high, the memory footprint will 
be tremendous, so Kylin add a check on this. If you see this error, suggest to 
identify the UHC dimension first and then re-evaluate the design (whether need 
to make that as dimension?). If must keep it, you can by-pass this error with 
couple ways:
+
+  change to use other encoding (like fixed_length, integer)
+  or set a bigger value for kylin.dictionary.max.cardinality in conf/kylin.properties.
+
+  
 
 
 3. Build cube failed due 
to âerror check statusâ

Modified: kylin/site/feed.xml
URL:

kylin git commit: update faq for EMR

2016-10-27 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document 1f3a45faa -> 3f00136ff


update faq for EMR


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f00136f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f00136f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f00136f

Branch: refs/heads/document
Commit: 3f00136ff48d3c7d36454f404cfb377c9a7ad017
Parents: 1f3a45f
Author: shaofengshi 
Authored: Fri Oct 28 10:25:58 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 28 10:25:58 2016 +0800

--
 website/_docs15/gettingstarted/faq.md | 36 --
 1 file changed, 29 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3f00136f/website/_docs15/gettingstarted/faq.md
--
diff --git a/website/_docs15/gettingstarted/faq.md 
b/website/_docs15/gettingstarted/faq.md
index e070159..9c1f906 100644
--- a/website/_docs15/gettingstarted/faq.md
+++ b/website/_docs15/gettingstarted/faq.md
@@ -10,21 +10,43 @@ since: v0.6.x
 
   * Kylin need many dependent jars (hadoop/hive/hcat/hbase/kafka) on classpath 
to work, but Kylin doesn't ship them. It will seek these jars from your local 
machine by running commands like `hbase classpath`, `hive -e set` etc. The 
founded jars' path will be appended to the environment variable 
*HBASE_CLASSPATH* (Kylin uses `hbase` shell command to start up, which will 
read this). But in some Hadoop distribution (like EMR 5.0), the `hbase` shell 
doesn't keep the origin `HBASE_CLASSPATH` value, that causes the 
"NoClassDefFoundError".
 
-  * To fix this, find the hbase shell script (in hbase/bin folder), and search 
*HBASE_CLASSPATH*, check whether it overwrite the value like :
+  To fix this, find the hbase shell script (in hbase/bin or hbase/conf 
folder), and search *HBASE_CLASSPATH*, check whether it overwrite the value 
like :
 
-  {% highlight Groff markup %}
+```
   export 
HBASE_CLASSPATH=$HADOOP_CONF:$HADOOP_HOME/*:$HADOOP_HOME/lib/*:$ZOOKEEPER_HOME/*:$ZOOKEEPER_HOME/lib/*
-  {% endhighlight %}
+```
 
-  * If true, change it to keep the origin value like:
+  If true, change it to keep the origin value like:
 
-   {% highlight Groff markup %}
+```
   export 
HBASE_CLASSPATH=$HADOOP_CONF:$HADOOP_HOME/*:$HADOOP_HOME/lib/*:$ZOOKEEPER_HOME/*:$ZOOKEEPER_HOME/lib/*:$HBASE_CLASSPATH
-  {% endhighlight %}
+```
+
+  * For EMR5.0, need change the hbase-env.sh
+
+```
+ sudo vi /usr/lib/hbase/conf/hbase-env.sh
+```
+
+ in around line 30 it was :
+
+```
+export HBASE_CLASSPATH=/etc/hadoop/conf
+```
+
+change to :
+
+```
+export HBASE_CLASSPATH=/etc/hadoop/conf:$HBASE_CLASSPATH
+```
+
+then restart Kylin.
 
  2. Get "java.lang.IllegalArgumentException: Too high cardinality is not 
suitable for dictionary -- cardinality: 5220674" in "Build Dimension 
Dictionary" step
 
-  * Kylin uses "Dictionary" encoding to encode/decode the dimension values 
(check [this blog](/blog/2015/08/13/kylin-dictionary/)); Usually a dimension's 
cardinality is less than millions, so the "Dict" encoding is good to use. As 
dictionary need be persisted and loaded into memory, if a dimension's 
cardinality is very high, the memory footprint will be tremendous, so Kylin add 
a check on this. If you see this error, suggest to identify the UHC dimension 
first and then re-evaluate the design (whether need to make that as 
dimension?). If must keep it, you can by-pass this error with couple ways: 1) 
change to use other encoding (like `fixed_length`, `integer`) 2) or set a 
bigger value for `kylin.dictionary.max.cardinality` in `conf/kylin.properties`.
+  * Kylin uses "Dictionary" encoding to encode/decode the dimension values 
(check [this blog](/blog/2015/08/13/kylin-dictionary/)); Usually a dimension's 
cardinality is less than millions, so the "Dict" encoding is good to use. As 
dictionary need be persisted and loaded into memory, if a dimension's 
cardinality is very high, the memory footprint will be tremendous, so Kylin add 
a check on this. If you see this error, suggest to identify the UHC dimension 
first and then re-evaluate the design (whether need to make that as 
dimension?). If must keep it, you can by-pass this error with couple ways: 
+* change to use other encoding (like `fixed_length`, `integer`) 
+* or set a bigger value for `kylin.dictionary.max.cardinality` in 
`conf/kylin.properties`.
 
  3. Build cube failed due to "error check status"

kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

2016-10-27 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 1af3d9ee3 -> 8199a90c5


KYLIN-2134 dont reuse Object[] returned to calcite


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8199a90c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8199a90c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8199a90c

Branch: refs/heads/v1.6.0-rc1
Commit: 8199a90c57df12e8ad4a19737ffa9aa17a69f3eb
Parents: 1af3d9e
Author: Li Yang 
Authored: Thu Oct 27 17:09:22 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Oct 28 09:51:51 2016 +0800

--
 .../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 4 
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java| 8 
 2 files changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8199a90c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 61267ae..00ba247 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -37,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Preconditions;
-
 public class SegmentCubeTupleIterator implements ITupleIterator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(SegmentCubeTupleIterator.class);
@@ -98,8 +96,6 @@ public class SegmentCubeTupleIterator implements 
ITupleIterator {
 }
 GTRecord curRecord = gtItr.next();
 
-Preconditions.checkNotNull(cubeTupleConverter);
-
 // translate into tuple
 advMeasureFillers = cubeTupleConverter.translateResult(curRecord, 
tuple);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8199a90c/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index dafbb5f..c7b3c71 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.query.enumerator;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Properties;
 
@@ -77,10 +78,9 @@ public class OLAPEnumerator implements Enumerator {
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {
-// make sure the tuple layout is correct
-//assert 
tuple.getAllFields().equals(olapContext.returnTupleInfo.getAllFields());
-
-current = tuple.getAllValues();
+// give calcite a new array every time, see details in KYLIN-2134
+Object[] values = tuple.getAllValues();
+current = Arrays.copyOf(values, values.length);
 return current;
 }

kylin git commit: KYLIN-1971 fix ITStorageTest

2016-10-27 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 3b9be1963 -> d06a425e3


KYLIN-1971 fix ITStorageTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d06a425e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d06a425e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d06a425e

Branch: refs/heads/master
Commit: d06a425e3b30b52ae895954a9faa034da1ad0e04
Parents: 3b9be19
Author: Li Yang 
Authored: Thu Oct 27 18:03:22 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 18:03:22 2016 +0800

--
 .../apache/kylin/metadata/model/TableRef.java   |  8 +++
 .../apache/kylin/storage/StorageMockUtils.java  | 52 +++-
 .../kylin/storage/hbase/ITStorageTest.java  | 18 +++
 3 files changed, 46 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d06a425e/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
index 254ce18..7089eba 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
@@ -70,6 +70,8 @@ public class TableRef {
 return Collections.unmodifiableCollection(columns.values());
 }
 
+// for test only
+@Deprecated
 public TblColRef makeFakeColumn(String name) {
 ColumnDesc colDesc = new ColumnDesc();
 colDesc.setName(name);
@@ -77,6 +79,12 @@ public class TableRef {
 return new TblColRef(this, colDesc);
 }
 
+// for test only
+@Deprecated
+public TblColRef makeFakeColumn(ColumnDesc colDesc) {
+return new TblColRef(this, colDesc);
+}
+
 @Override
 public boolean equals(Object o) {
 if (this == o)

http://git-wip-us.apache.org/repos/asf/kylin/blob/d06a425e/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
--
diff --git 
a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java 
b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
index 456e768..0786f32 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
@@ -28,9 +28,11 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.TupleInfo;
 
@@ -39,7 +41,14 @@ import com.google.common.collect.ImmutableList;
 /**
  */
 public class StorageMockUtils {
-public static TupleInfo newTupleInfo(List groups, 
List aggregations) {
+
+final DataModelDesc model;
+
+public StorageMockUtils(DataModelDesc model) {
+this.model = model;
+}
+
+public TupleInfo newTupleInfo(List groups, List 
aggregations) {
 TupleInfo info = new TupleInfo();
 int idx = 0;
 
@@ -47,34 +56,32 @@ public class StorageMockUtils {
 info.setField(col.getName(), col, idx++);
 }
 
-TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
+TableRef sourceTable = groups.get(0).getTableRef();
 for (FunctionDesc func : aggregations) {
-TblColRef col = func.newFakeRewriteColumn(sourceTable).getRef();
+ColumnDesc colDesc = 
func.newFakeRewriteColumn(sourceTable.getTableDesc());
+TblColRef col = sourceTable.makeFakeColumn(colDesc);
 info.setField(col.getName(), col, idx++);
 }
 
 return info;
 }
 
-public static List buildGroups() {
+public List buildGroups() {
 List groups = new ArrayList();
 
-TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
-TblColRef c1 = TblColRef.mockup(t1, 2, "CAL_DT", "date");
+TblColRef c1 = model.findColumn("DEFAULT.TEST_KYLIN_FACT.CAL_DT");
 groups.add(c1);
 
-TableDesc t2 = TableDesc.mockup("DEFAULT.TEST_CATEGORY_GROUPINGS");
-TblColRef c2 =

kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

2016-10-27 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang21 5e3ea3c13 -> 5d1db92be


KYLIN-2134 dont reuse Object[] returned to calcite


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d1db92b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d1db92b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d1db92b

Branch: refs/heads/yang21
Commit: 5d1db92bea3bc05f02ca4e411100d58526a51cdf
Parents: 5e3ea3c
Author: Li Yang 
Authored: Thu Oct 27 17:09:22 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 17:10:53 2016 +0800

--
 .../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 4 
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java| 8 
 2 files changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5d1db92b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 61267ae..00ba247 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -37,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Preconditions;
-
 public class SegmentCubeTupleIterator implements ITupleIterator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(SegmentCubeTupleIterator.class);
@@ -98,8 +96,6 @@ public class SegmentCubeTupleIterator implements 
ITupleIterator {
 }
 GTRecord curRecord = gtItr.next();
 
-Preconditions.checkNotNull(cubeTupleConverter);
-
 // translate into tuple
 advMeasureFillers = cubeTupleConverter.translateResult(curRecord, 
tuple);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d1db92b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index dafbb5f..c7b3c71 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.query.enumerator;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Properties;
 
@@ -77,10 +78,9 @@ public class OLAPEnumerator implements Enumerator {
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {
-// make sure the tuple layout is correct
-//assert 
tuple.getAllFields().equals(olapContext.returnTupleInfo.getAllFields());
-
-current = tuple.getAllValues();
+// give calcite a new array every time, see details in KYLIN-2134
+Object[] values = tuple.getAllValues();
+current = Arrays.copyOf(values, values.length);
 return current;
 }

kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

2016-10-27 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 49260dd0c -> 3b9be1963


KYLIN-2134 dont reuse Object[] returned to calcite


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3b9be196
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3b9be196
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3b9be196

Branch: refs/heads/master
Commit: 3b9be19634c8f295a9c7468bcdaf8ab2aad73fcb
Parents: 49260dd
Author: Li Yang 
Authored: Thu Oct 27 17:09:22 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 17:09:22 2016 +0800

--
 .../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 4 
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java| 8 
 2 files changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3b9be196/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 61267ae..00ba247 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -37,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Preconditions;
-
 public class SegmentCubeTupleIterator implements ITupleIterator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(SegmentCubeTupleIterator.class);
@@ -98,8 +96,6 @@ public class SegmentCubeTupleIterator implements 
ITupleIterator {
 }
 GTRecord curRecord = gtItr.next();
 
-Preconditions.checkNotNull(cubeTupleConverter);
-
 // translate into tuple
 advMeasureFillers = cubeTupleConverter.translateResult(curRecord, 
tuple);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3b9be196/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index dafbb5f..c7b3c71 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.query.enumerator;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Properties;
 
@@ -77,10 +78,9 @@ public class OLAPEnumerator implements Enumerator {
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {
-// make sure the tuple layout is correct
-//assert 
tuple.getAllFields().equals(olapContext.returnTupleInfo.getAllFields());
-
-current = tuple.getAllValues();
+// give calcite a new array every time, see details in KYLIN-2134
+Object[] values = tuple.getAllValues();
+current = Arrays.copyOf(values, values.length);
 return current;
 }

kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

2016-10-27 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/1.5.4.1-beeline 8f4ed7279 -> 3fb0c16c2


KYLIN-2134 Kylin will treat empty string as NULL by mistake


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3fb0c16c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3fb0c16c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3fb0c16c

Branch: refs/heads/1.5.4.1-beeline
Commit: 3fb0c16c263d8a7edc443d919200f0c244c517fb
Parents: 8f4ed72
Author: Hongbin Ma 
Authored: Thu Oct 27 15:22:54 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 15:25:31 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3fb0c16c/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index aaf9aa9..9074403 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -178,7 +178,7 @@ public class Tuple implements ITuple {
 if (strValue == null)
 return null;
 
-if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string"))
+if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
 return null;
 
 // TODO use data type enum instead of string comparison

kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

2016-10-27 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 cd2a06a5d -> 1af3d9ee3


KYLIN-2134 Kylin will treat empty string as NULL by mistake


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1af3d9ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1af3d9ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1af3d9ee

Branch: refs/heads/v1.6.0-rc1
Commit: 1af3d9ee31f3edd64eea336c65b7619b078efdbf
Parents: cd2a06a
Author: Hongbin Ma 
Authored: Thu Oct 27 15:22:54 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 15:25:48 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1af3d9ee/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index aaf9aa9..9074403 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -178,7 +178,7 @@ public class Tuple implements ITuple {
 if (strValue == null)
 return null;
 
-if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string"))
+if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
 return null;
 
 // TODO use data type enum instead of string comparison

kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

2016-10-27 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21 9287117e1 -> 5e3ea3c13


KYLIN-2134 Kylin will treat empty string as NULL by mistake


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5e3ea3c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5e3ea3c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5e3ea3c1

Branch: refs/heads/yang21
Commit: 5e3ea3c13c52513bcfa6781674424276b93bb51d
Parents: 9287117
Author: Hongbin Ma 
Authored: Thu Oct 27 15:22:54 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 15:26:07 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5e3ea3c1/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index aaf9aa9..9074403 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -178,7 +178,7 @@ public class Tuple implements ITuple {
 if (strValue == null)
 return null;
 
-if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string"))
+if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
 return null;
 
 // TODO use data type enum instead of string comparison

kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

2016-10-27 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 468fe9ffc -> 49260dd0c


KYLIN-2134 Kylin will treat empty string as NULL by mistake


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49260dd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49260dd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49260dd0

Branch: refs/heads/master
Commit: 49260dd0cb1c129d40b51c359197402441b87256
Parents: 468fe9f
Author: Hongbin Ma 
Authored: Thu Oct 27 15:22:54 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 15:22:57 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/49260dd0/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index aaf9aa9..9074403 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -178,7 +178,7 @@ public class Tuple implements ITuple {
 if (strValue == null)
 return null;
 
-if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string"))
+if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
 return null;
 
 // TODO use data type enum instead of string comparison

kylin git commit: minor, drop kylin.job.step.timeout

2016-10-27 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang21 2d87faec4 -> 9287117e1


minor, drop kylin.job.step.timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9287117e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9287117e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9287117e

Branch: refs/heads/yang21
Commit: 9287117e17bcd5d9f7c7b93bfa5b5d3b2ad7b0c3
Parents: 2d87fae
Author: Li Yang 
Authored: Thu Oct 27 15:11:20 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 15:11:20 2016 +0800

--
 build/conf/kylin_job_conf_inmem.xml   | 4 ++--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java| 4 
 .../java/org/apache/kylin/job/engine/JobEngineConfig.java | 7 ---
 3 files changed, 2 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9287117e/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index 73b5cc5..d6799d5 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -81,8 +81,8 @@
 
 
 mapred.task.timeout
-360
-Set task timeout to 1 hour
+720
+Set task timeout to 2 hour
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9287117e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 53074fd..f824d8e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -407,10 +407,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptionalStringArray("kylin.job.admin.dls", null);
 }
 
-public long getJobStepTimeout() {
-return Long.parseLong(getOptional("kylin.job.step.timeout", 
String.valueOf(2 * 60 * 60)));
-}
-
 public double getJobCuboidSizeRatio() {
 return Double.parseDouble(getOptional("kylin.job.cuboid.size.ratio", 
"0.25"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9287117e/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java 
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 8ba1730..1e3eeaa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -145,13 +145,6 @@ public class JobEngineConfig {
 }
 
 /**
- * @return the jobStepTimeout
- */
-public long getJobStepTimeout() {
-return config.getJobStepTimeout();
-}
-
-/**
  * @return the asyncJobCheckInterval
  */
 public int getAsyncJobCheckInterval() {

kylin git commit: minor, drop kylin.job.step.timeout

2016-10-27 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master a43729bc7 -> 468fe9ffc


minor, drop kylin.job.step.timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/468fe9ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/468fe9ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/468fe9ff

Branch: refs/heads/master
Commit: 468fe9ffc3d1b6c6813481803dd59673f78718f1
Parents: a43729b
Author: Li Yang 
Authored: Thu Oct 27 15:01:51 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 15:01:51 2016 +0800

--
 build/conf/kylin_job_conf_inmem.xml   | 2 +-
 .../main/java/org/apache/kylin/common/KylinConfigBase.java| 4 
 .../java/org/apache/kylin/job/engine/JobEngineConfig.java | 7 ---
 3 files changed, 1 insertion(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/468fe9ff/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index 1d349f6..d363507 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -81,7 +81,7 @@
 
 
 mapreduce.task.timeout
-360
+720
 Set task timeout to 1 hour
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/468fe9ff/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 455b691..8744296 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -411,10 +411,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptionalStringArray("kylin.job.admin.dls", null);
 }
 
-public long getJobStepTimeout() {
-return Long.parseLong(getOptional("kylin.job.step.timeout", 
String.valueOf(2 * 60 * 60)));
-}
-
 public double getJobCuboidSizeRatio() {
 return Double.parseDouble(getOptional("kylin.job.cuboid.size.ratio", 
"0.25"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/468fe9ff/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java 
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 8ba1730..1e3eeaa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -145,13 +145,6 @@ public class JobEngineConfig {
 }
 
 /**
- * @return the jobStepTimeout
- */
-public long getJobStepTimeout() {
-return config.getJobStepTimeout();
-}
-
-/**
  * @return the asyncJobCheckInterval
  */
 public int getAsyncJobCheckInterval() {

kylin git commit: minor, remove kylin.dictionary.max.cardinality

2016-10-27 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang21 b502a1741 -> 2d87faec4


minor, remove kylin.dictionary.max.cardinality


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2d87faec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2d87faec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2d87faec

Branch: refs/heads/yang21
Commit: 2d87faec4d04e842ea73bb0493209b661305a4ee
Parents: b502a17
Author: Li Yang 
Authored: Thu Oct 27 14:34:40 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 14:35:43 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java |  4 
 .../org/apache/kylin/dict/DictionaryGenerator.java| 14 --
 2 files changed, 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2d87faec/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5d92aef..53074fd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -448,10 +448,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.cube.measure.customMeasureType.");
 }
 
-public int getDictionaryMaxCardinality() {
-return 
Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "3000"));
-}
-
 public int getTableSnapshotMaxMB() {
 return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", 
"300"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2d87faec/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index ba848c6..0adf40e 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
@@ -40,20 +39,10 @@ import com.google.common.base.Preconditions;
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class DictionaryGenerator {
 
-private static final int DICT_MAX_CARDINALITY = 
getDictionaryMaxCardinality();
-
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryGenerator.class);
 
 private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
 
-private static int getDictionaryMaxCardinality() {
-try {
-return 
KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
-} catch (Throwable e) {
-return 3000; // some test case does not have KylinConfig setup 
properly
-}
-}
-
 public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
 Preconditions.checkNotNull(dataType, "dataType cannot be null");
 
@@ -92,9 +81,6 @@ public class DictionaryGenerator {
 logger.debug("Dictionary cardinality: " + dict.getSize());
 logger.debug("Dictionary builder class: " + 
builder.getClass().getName());
 logger.debug("Dictionary class: " + dict.getClass().getName());
-if (dict instanceof TrieDictionary && dict.getSize() > 
DICT_MAX_CARDINALITY) {
-throw new IllegalArgumentException("Too high cardinality is not 
suitable for dictionary -- cardinality: " + dict.getSize());
-}
 return dict;
 }

kylin git commit: minor, remove kylin.dictionary.max.cardinality

2016-10-27 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 5c1c24a9f -> a43729bc7


minor, remove kylin.dictionary.max.cardinality


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a43729bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a43729bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a43729bc

Branch: refs/heads/master
Commit: a43729bc75b431bbabee49e89a4e60c60cbf6baa
Parents: 5c1c24a
Author: Li Yang 
Authored: Thu Oct 27 14:34:40 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 14:34:40 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java |  4 
 .../org/apache/kylin/dict/DictionaryGenerator.java| 14 --
 2 files changed, 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a43729bc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f49127b..455b691 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -452,10 +452,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.cube.measure.customMeasureType.");
 }
 
-public int getDictionaryMaxCardinality() {
-return 
Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "3000"));
-}
-
 public int getTableSnapshotMaxMB() {
 return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", 
"300"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a43729bc/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index ba848c6..0adf40e 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
@@ -40,20 +39,10 @@ import com.google.common.base.Preconditions;
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class DictionaryGenerator {
 
-private static final int DICT_MAX_CARDINALITY = 
getDictionaryMaxCardinality();
-
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryGenerator.class);
 
 private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
 
-private static int getDictionaryMaxCardinality() {
-try {
-return 
KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
-} catch (Throwable e) {
-return 3000; // some test case does not have KylinConfig setup 
properly
-}
-}
-
 public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
 Preconditions.checkNotNull(dataType, "dataType cannot be null");
 
@@ -92,9 +81,6 @@ public class DictionaryGenerator {
 logger.debug("Dictionary cardinality: " + dict.getSize());
 logger.debug("Dictionary builder class: " + 
builder.getClass().getName());
 logger.debug("Dictionary class: " + dict.getClass().getName());
-if (dict instanceof TrieDictionary && dict.getSize() > 
DICT_MAX_CARDINALITY) {
-throw new IllegalArgumentException("Too high cardinality is not 
suitable for dictionary -- cardinality: " + dict.getSize());
-}
 return dict;
 }

[2/2] kylin git commit: KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as QueryId

svn commit: r1766917 - in /kylin/site: docs15/gettingstarted/faq.html feed.xml

kylin git commit: update faq for EMR

kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

kylin git commit: KYLIN-1971 fix ITStorageTest

kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

kylin git commit: minor, drop kylin.job.step.timeout

kylin git commit: minor, drop kylin.job.step.timeout

kylin git commit: minor, remove kylin.dictionary.max.cardinality

kylin git commit: minor, remove kylin.dictionary.max.cardinality

15 matches

Site Navigation

Mail list logo

Footer information