subject:"kylin git commit\: minor"

[4/4] kylin git commit: Minor, make the sample kylin_streaming_cube be consistent with kylin_sales

2018-02-09 Thread billyliu

Minor, make the sample kylin_streaming_cube be consistent with kylin_sales

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4b25ec8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4b25ec8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4b25ec8

Branch: refs/heads/2.3.x
Commit: b4b25ec8cec9750b7a671c960448dd9feed56f1a
Parents: bd88a71
Author: shaofengshi 
Authored: Tue Feb 6 11:28:42 2018 +0800
Committer: Billy Liu 
Committed: Fri Feb 9 21:25:19 2018 +0800

--
 build/bin/sample.sh  | 2 ++
 .../sample_cube/template/cube_desc/kylin_streaming_cube.json | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b4b25ec8/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 0809d2b..18823ae 100755
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -87,6 +87,8 @@ cp -rf ${KYLIN_HOME}/sample_cube/template/* 
${KYLIN_HOME}/sample_cube/metadata
 
 sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_streaming_cube.json
+sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_streaming_cube.json
 
  Add version info into cubes
 kylin_version_str=`bash ${KYLIN_HOME}/bin/kylin.sh version | grep 
kylin.version`

http://git-wip-us.apache.org/repos/asf/kylin/blob/b4b25ec8/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
index 2bd5584..21466e0 100644
--- a/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
@@ -145,7 +145,7 @@
   "status_need_notify" : [ ],
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
-  "engine_type" : 2,
-  "storage_type" : 2,
+  "engine_type" : %default_engine_type%,
+  "storage_type" : %default_storage_type%,
   "partition_date_start": 0
 }
\ No newline at end of file

[3/5] kylin git commit: minor, tell prepared SQL in log summary

2018-02-03 Thread liyang

minor, tell prepared SQL in log summary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ade239f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ade239f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ade239f

Branch: refs/heads/master
Commit: 4ade239fd0ba027fda70b2e61f2a510ee31fbaea
Parents: da14ee9
Author: Yifan Zhang 
Authored: Wed Jan 31 00:03:31 2018 +0800
Committer: Li Yang 
Committed: Sun Feb 4 08:45:09 2018 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/QueryService.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4ade239f/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 841cad9..bbf71c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -323,6 +323,7 @@ public class QueryService extends BasicService {
 stringBuilder.append("Hit Exception Cache: 
").append(response.isHitExceptionCache()).append(newLine);
 stringBuilder.append("Storage cache used: 
").append(storageCacheUsed).append(newLine);
 stringBuilder.append("Is Query Push-Down: 
").append(isPushDown).append(newLine);
+stringBuilder.append("Is Prepare: 
").append(BackdoorToggles.getPrepareOnly()).append(newLine);
 stringBuilder.append("Trace URL: 
").append(response.getTraceUrl()).append(newLine);
 stringBuilder.append("Message: 
").append(response.getExceptionMessage()).append(newLine);
 
stringBuilder.append("==[QUERY]===").append(newLine);

[5/5] kylin git commit: minor, fix DefaultSchedulerTest.testMetaStoreRecover()

2018-02-03 Thread liyang

minor, fix DefaultSchedulerTest.testMetaStoreRecover()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76dc0497
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76dc0497
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76dc0497

Branch: refs/heads/master
Commit: 76dc049717632eaf1f4c80e58ce61fa6081ae24f
Parents: 04b3220
Author: Li Yang 
Authored: Sun Feb 4 09:43:53 2018 +0800
Committer: Li Yang 
Committed: Sun Feb 4 09:43:53 2018 +0800

--
 .../apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/76dc0497/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index d1b7d96..c7c69cd 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -175,15 +175,15 @@ public class DefaultSchedulerTest extends 
BaseSchedulerTest {
 }
 
 @Test
-public void tesMetaStoreRecover() throws Exception {
+public void testMetaStoreRecover() throws Exception {
 logger.info("tesMetaStoreRecover");
 NoErrorStatusExecutable job = new NoErrorStatusExecutable();
 ErrorTestExecutable task = new ErrorTestExecutable();
 job.addTask(task);
 execMgr.addJob(job);
-Thread.sleep(2000);
+Thread.sleep(2500);
 runningJobToError(job.getId());
-Thread.sleep(2000);
+Thread.sleep(2500);
 Assert.assertEquals(ExecutableState.ERROR, 
execMgr.getOutput(job.getId()).getState());
 }

kylin git commit: minor, fix DefaultSchedulerTest.testMetaStoreRecover()

2018-02-03 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/sync 04b322073 -> 76dc04971


minor, fix DefaultSchedulerTest.testMetaStoreRecover()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76dc0497
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76dc0497
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76dc0497

Branch: refs/heads/sync
Commit: 76dc049717632eaf1f4c80e58ce61fa6081ae24f
Parents: 04b3220
Author: Li Yang 
Authored: Sun Feb 4 09:43:53 2018 +0800
Committer: Li Yang 
Committed: Sun Feb 4 09:43:53 2018 +0800

--
 .../apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/76dc0497/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index d1b7d96..c7c69cd 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -175,15 +175,15 @@ public class DefaultSchedulerTest extends 
BaseSchedulerTest {
 }
 
 @Test
-public void tesMetaStoreRecover() throws Exception {
+public void testMetaStoreRecover() throws Exception {
 logger.info("tesMetaStoreRecover");
 NoErrorStatusExecutable job = new NoErrorStatusExecutable();
 ErrorTestExecutable task = new ErrorTestExecutable();
 job.addTask(task);
 execMgr.addJob(job);
-Thread.sleep(2000);
+Thread.sleep(2500);
 runningJobToError(job.getId());
-Thread.sleep(2000);
+Thread.sleep(2500);
 Assert.assertEquals(ExecutableState.ERROR, 
execMgr.getOutput(job.getId()).getState());
 }

[49/50] [abbrv] kylin git commit: minor, tell prepared SQL in log summary

2018-02-03 Thread liyang

minor, tell prepared SQL in log summary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ade239f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ade239f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ade239f

Branch: refs/heads/sync
Commit: 4ade239fd0ba027fda70b2e61f2a510ee31fbaea
Parents: da14ee9
Author: Yifan Zhang 
Authored: Wed Jan 31 00:03:31 2018 +0800
Committer: Li Yang 
Committed: Sun Feb 4 08:45:09 2018 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/QueryService.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4ade239f/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 841cad9..bbf71c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -323,6 +323,7 @@ public class QueryService extends BasicService {
 stringBuilder.append("Hit Exception Cache: 
").append(response.isHitExceptionCache()).append(newLine);
 stringBuilder.append("Storage cache used: 
").append(storageCacheUsed).append(newLine);
 stringBuilder.append("Is Query Push-Down: 
").append(isPushDown).append(newLine);
+stringBuilder.append("Is Prepare: 
").append(BackdoorToggles.getPrepareOnly()).append(newLine);
 stringBuilder.append("Trace URL: 
").append(response.getTraceUrl()).append(newLine);
 stringBuilder.append("Message: 
").append(response.getExceptionMessage()).append(newLine);
 
stringBuilder.append("==[QUERY]===").append(newLine);

[37/50] [abbrv] kylin git commit: minor, stablize org.apache.kylin.dict.TrieDictionaryForestTest#englishWordsTest

2018-02-03 Thread liyang

minor, stablize org.apache.kylin.dict.TrieDictionaryForestTest#englishWordsTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eedd135
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eedd135
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eedd135

Branch: refs/heads/sync
Commit: 5eedd135d27da96c2242079fd9e188fce0518f90
Parents: 9198cfe
Author: lidongsjtu 
Authored: Sun Jan 28 19:55:07 2018 +0800
Committer: lidongsjtu 
Committed: Sun Jan 28 19:55:07 2018 +0800

--
 .../kylin/dict/TrieDictionaryForestTest.java| 28 ++--
 1 file changed, 20 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eedd135/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
 
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 099075c..1f63111 100644
--- 
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ 
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -43,6 +43,7 @@ import java.util.NoSuchElementException;
 import java.util.Random;
 import java.util.TreeSet;
 
+import org.apache.kylin.common.util.Bytes;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -187,8 +188,14 @@ public class TrieDictionaryForestTest {
 str.add("party");
 str.add("parties");
 str.add("paint");
-String longStr = 
"paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
-+ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
+String longStr = 
"paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
 System.out.println("The length of the long string is " + 
longStr.length());
 str.add(longStr);
 
@@ -740,7 +747,8 @@ public class TrieDictionaryForestTest {
 System.out.println("times:" + i);
 }
 
-System.out.println("compare build time.  Old trie : " + 
oldDictTotalBuildTime / 1000.0 + "s.New trie : " + newDictTotalBuildTime / 
1000.0 + "s");
+System.out.println("compare build time.  Old trie : " + 
oldDictTotalBuildTime / 1000.0 + "s.New trie : "
++ newDictTotalBuildTime / 1000.0 + "s");
 }
 
 private void

[23/50] [abbrv] kylin git commit: minor, refine basic tuple filter class

2018-02-03 Thread liyang

minor, refine basic tuple filter class


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2b69155
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2b69155
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2b69155

Branch: refs/heads/sync
Commit: c2b69155215f72a5f78bf65995390431ff424c1e
Parents: 1f33dd9
Author: Roger Shi 
Authored: Mon Jan 8 17:57:57 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../metadata/filter/LogicalTupleFilter.java | 26 
 .../kylin/metadata/filter/TupleFilter.java  | 43 +
 .../kylin/metadata/filter/TupleFilterTest.java  | 65 
 3 files changed, 134 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2b69155/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 7893ed8..f0c825f 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -170,4 +170,30 @@ public class LogicalTupleFilter extends TupleFilter 
implements IOptimizeableTupl
 return transformer.visit(this);
 }
 
+@Override
+public boolean equals(Object other) {
+if (other == this) {
+return true;
+}
+if (!(other instanceof LogicalTupleFilter)) {
+return false;
+}
+final LogicalTupleFilter otherFilter = (LogicalTupleFilter) other;
+if (otherFilter.operator != this.operator || 
otherFilter.children.size() != this.children.size()) {
+return false;
+}
+
+for (int i = 0; i < otherFilter.children.size(); i++) {
+if (!otherFilter.children.get(i).equals(this.children.get(i))) {
+return false;
+}
+}
+
+return true;
+}
+
+@Override
+public int hashCode() {
+return (operator == null ? 0 : operator.hashCode()) + 31 * 
this.children.hashCode();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2b69155/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 5ba8726..09b41f5 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -99,6 +99,11 @@ public abstract class TupleFilter {
 addChild(c); // subclass overrides addChild()
 }
 
+final public void addChildren(TupleFilter... children) {
+for (TupleFilter c : children)
+addChild(c); // subclass overrides addChild()
+}
+
 public List getChildren() {
 return children;
 }
@@ -121,6 +126,44 @@ public abstract class TupleFilter {
 }
 
 /**
+ * The storage level dislike NOT logic
+ */
+public TupleFilter removeNot() {
+return removeNotInternal(this);
+}
+
+private TupleFilter removeNotInternal(TupleFilter filter) {
+FilterOperatorEnum op = filter.getOperator();
+
+if (!(filter instanceof LogicalTupleFilter)) {
+return filter;
+}
+
+LogicalTupleFilter logicalFilter = (LogicalTupleFilter) filter;
+
+switch (logicalFilter.operator) {
+case NOT:
+assert (filter.children.size() == 1);
+TupleFilter reverse = filter.children.get(0).reverse();
+return removeNotInternal(reverse);
+case AND:
+LogicalTupleFilter andFilter = new 
LogicalTupleFilter(FilterOperatorEnum.AND);
+for (TupleFilter child : logicalFilter.children) {
+andFilter.addChild(removeNotInternal(child));
+}
+return andFilter;
+case OR:
+LogicalTupleFilter orFilter = new 
LogicalTupleFilter(FilterOperatorEnum.OR);
+for (TupleFilter child : logicalFilter.children) {
+orFilter.addChild(removeNotInternal(child));
+}
+return orFilter;
+default:
+throw new IllegalStateException("This filter is unexpected: " + 
filter);
+}
+}
+
+/**
  * flatten to OR-AND filter, (A AND B AND ..) OR (C AND D AND ..) OR ..

[05/50] [abbrv] kylin git commit: minor refactor

2018-02-03 Thread liyang

minor refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0988c846
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0988c846
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0988c846

Branch: refs/heads/sync
Commit: 0988c846823ca65a46eb6ab9b3dcee443da57872
Parents: 0ae39b5
Author: Hongbin Ma 
Authored: Thu Jan 11 16:47:37 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../java/org/apache/kylin/rest/request/ModelRequest.java  | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0988c846/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java 
b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
index 5f07997..2c5b3bf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
@@ -26,6 +26,8 @@ public class ModelRequest {
 private boolean successful;
 private String message;
 private String project;
+
+private String ccInCheck;
 private boolean isSeekingExprAdvice;
 
 public String getUuid() {
@@ -112,4 +114,12 @@ public class ModelRequest {
 public void setIsSeekingExprAdvice(boolean seekingExprAdvice) {
 isSeekingExprAdvice = seekingExprAdvice;
 }
+
+public String getCcInCheck() {
+return ccInCheck;
+}
+
+public void setCcInCheck(String ccInCheck) {
+this.ccInCheck = ccInCheck;
+}
 }

[19/50] [abbrv] kylin git commit: minor, add switch table-dir-create-first for build cube with hive step.

2018-02-03 Thread liyang

minor, add switch table-dir-create-first for build cube with hive step.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce1775a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce1775a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce1775a5

Branch: refs/heads/sync
Commit: ce1775a5ab0e775bce6685e7801f114a4aff9b9b
Parents: 543124d
Author: yiming.xu <100650...@qq.com>
Authored: Wed Jan 24 17:33:18 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java   | 4 
 .../main/java/org/apache/kylin/source/hive/HiveMRInput.java  | 8 +---
 .../java/org/apache/kylin/source/hive/HiveMRInputTest.java   | 2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5045b2f..6d1b7f9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -755,6 +755,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.source.hive.sparksql-beeline-params", "");
 }
 
+public boolean getHiveTableDirCreateFirst() {
+return 
Boolean.parseBoolean(getOptional("kylin.source.hive.table-dir-create-first", 
"false"));
+}
+
 public String getFlatHiveTableClusterByDictColumn() {
 return 
getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 31a99cd..f02d2ea 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -179,9 +179,11 @@ public class HiveMRInput implements IMRInput {
 protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
 
 String jobWorkingDir = 
JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
-// Create work dir to avoid hive create it,
-// the difference is that the owners are different.
-checkAndCreateWorkDir(jobWorkingDir);
+if (KylinConfig.getInstanceFromEnv().getHiveTableDirCreateFirst()) 
{
+// Create work dir to avoid hive create it,
+// the difference is that the owners are different.
+checkAndCreateWorkDir(jobWorkingDir);
+}
 return jobWorkingDir;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
--
diff --git 
a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java 
b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index 6d0737d..f33ee74 100644
--- 
a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ 
b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -41,8 +41,8 @@ public class HiveMRInputTest {
 try {
 KylinConfig kylinConfig = mock(KylinConfig.class);
 KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true);
 
when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
-
 DefaultChainedExecutable defaultChainedExecutable = 
mock(DefaultChainedExecutable.class);
 defaultChainedExecutable.setId(UUID.randomUUID().toString());

[13/50] [abbrv] kylin git commit: minor, restore OLAPToEnumerableConverter to original

2018-02-03 Thread liyang

minor, restore OLAPToEnumerableConverter to original


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a4acb4d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a4acb4d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a4acb4d5

Branch: refs/heads/sync
Commit: a4acb4d53ef74f8e59f46f550143c599c546ff30
Parents: f73219c
Author: Roger Shi 
Authored: Wed Jan 3 22:21:28 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../apache/calcite/plan/OLAPRelMdRowCount.java  | 174 
 .../calcite/plan/OLAPRelMetadataProvider.java   |  39 
 .../org/apache/calcite/plan/RelOptCluster.java  | 199 ---
 .../src/test/resources/query/sql/query110.sql   |  58 ++
 .../relnode/OLAPToEnumerableConverter.java  |   2 +-
 5 files changed, 59 insertions(+), 413 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a4acb4d5/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java 
b/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
deleted file mode 100644
index 751561f..000
--- a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
+++ /dev/null
@@ -1,174 +0,0 @@
-package org.apache.calcite.plan;
-
-import org.apache.calcite.adapter.enumerable.EnumerableLimit;
-import org.apache.calcite.plan.volcano.RelSubset;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.SingleRel;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.Calc;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Intersect;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.Minus;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.SemiJoin;
-import org.apache.calcite.rel.core.Sort;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.core.Union;
-import org.apache.calcite.rel.core.Values;
-import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMdRowCount;
-import org.apache.calcite.rel.metadata.RelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.util.BuiltInMethod;
-
-public class OLAPRelMdRowCount extends RelMdRowCount {
-public static final RelMetadataProvider SOURCE = 
ReflectiveRelMetadataProvider
-.reflectiveSource(BuiltInMethod.ROW_COUNT.method, new 
OLAPRelMdRowCount());
-
-//avoid case like sql/query60.sql, which will generate plan:
-/*
-EnumerableLimit(fetch=[3])
-OLAPToEnumerableConverter
-OLAPAggregateRel(group=[{0}], SUM_PRICE=[SUM($1)], 
CNT_1=[COUNT()], TOTAL_ITEMS=[SUM($2)])
- */
-private boolean shouldIntercept(RelNode rel) {
-for (RelNode input : rel.getInputs()) {
-if (input instanceof RelSubset) {
-RelSubset relSubset = (RelSubset) input;
-if (relSubset.getBest() != null
-&& 
relSubset.getBest().getClass().getCanonicalName().endsWith("OLAPToEnumerableConverter"))
 {
-return true;
-}
-}
-}
-return false;
-}
-
-@Override
-public Double getRowCount(RelNode rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(RelSubset subset, RelMetadataQuery mq) {
-if (shouldIntercept(subset))
-return 1E10;
-
-return super.getRowCount(subset, mq);
-}
-
-@Override
-public Double getRowCount(Union rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Intersect rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Minus rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Filter rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Calc rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-

[36/50] [abbrv] kylin git commit: minor, fix some typos

2018-02-03 Thread liyang

minor, fix some typos


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9198cfe8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9198cfe8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9198cfe8

Branch: refs/heads/sync
Commit: 9198cfe8f1439246eff6abff38b3869453325aca
Parents: 74e3f61
Author: lidongsjtu 
Authored: Sun Jan 28 19:43:00 2018 +0800
Committer: lidongsjtu 
Committed: Sun Jan 28 19:43:00 2018 +0800

--
 .../java/org/apache/kylin/source/hive/HiveMetadataExplorer.java  | 4 ++--
 .../src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9198cfe8/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index cb3eb02..9d4cc53 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -185,7 +185,7 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 @Override
 public ColumnDesc[] evalQueryMetadata(String query) {
 if (StringUtils.isEmpty(query)) {
-throw new RuntimeException("Evalutate query shall not be empty.");
+throw new RuntimeException("Evaluate query shall not be empty.");
 }
 
 KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -204,7 +204,7 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 HiveTableMeta hiveTableMeta = 
hiveClient.getHiveTableMeta(tmpDatabase, tmpView);
 return extractColumnFromMeta(hiveTableMeta);
 } catch (Exception e) {
-throw new RuntimeException("Cannot evalutate metadata of query: " 
+ query, e);
+throw new RuntimeException("Cannot evaluate metadata of query: " + 
query, e);
 } finally {
 try {
 logger.debug("Cleaning up.");

http://git-wip-us.apache.org/repos/asf/kylin/blob/9198cfe8/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java 
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index d96a68d..c34597a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -232,7 +232,7 @@ public class JdbcExplorer implements 
ISourceMetadataExplorer, ISampleDataDeploye
 @Override
 public ColumnDesc[] evalQueryMetadata(String query) {
 if (StringUtils.isEmpty(query)) {
-throw new RuntimeException("Evalutate query shall not be empty.");
+throw new RuntimeException("Evaluate query shall not be empty.");
 }
 
 KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -256,7 +256,7 @@ public class JdbcExplorer implements 
ISourceMetadataExplorer, ISampleDataDeploye
 DBUtils.closeQuietly(con);
 return result;
 } catch (Exception e) {
-throw new RuntimeException("Cannot evalutate metadata of query: " 
+ query, e);
+throw new RuntimeException("Cannot evaluate metadata of query: " + 
query, e);
 } finally {
 try {
 logger.debug("Cleaning up.");

[21/50] [abbrv] kylin git commit: minor refactor: avoid exposing cubedesc's getError public api

2018-02-03 Thread liyang

minor refactor: avoid exposing cubedesc's getError public api


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59d50646
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59d50646
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59d50646

Branch: refs/heads/sync
Commit: 59d5064673282ba36783e0796523ad83c2dc5e8c
Parents: 16236ef
Author: Hongbin Ma 
Authored: Fri Jan 26 19:13:12 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../org/apache/kylin/cube/CubeDescManager.java  |  2 +-
 .../org/apache/kylin/cube/CubeInstance.java | 29 ++
 .../org/apache/kylin/cube/model/CubeDesc.java   | 13 ++--
 .../kylin/rest/controller/CubeController.java   | 32 
 .../apache/kylin/rest/service/CubeService.java  | 12 
 5 files changed, 47 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59d50646/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index cd43f15..1db66e1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -198,7 +198,7 @@ public class CubeDescManager {
 
 postProcessCubeDesc(cubeDesc);
 // Check base validation
-if (!cubeDesc.getError().isEmpty()) {
+if (cubeDesc.isBroken()) {
 return cubeDesc;
 }
 // Semantic validation

http://git-wip-us.apache.org/repos/asf/kylin/blob/59d50646/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index a299203..6395abd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -6,15 +6,15 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
-*/
+ */
 
 package org.apache.kylin.cube;
 
@@ -130,13 +130,13 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // default constructor for jackson
 public CubeInstance() {
 }
-
+
 public CubeInstance latestCopyForWrite() {
 CubeManager mgr = CubeManager.getInstance(config);
 CubeInstance latest = mgr.getCube(name); // in case this object is 
out-of-date
 return mgr.copyForWrite(latest);
 }
-
+
 void init(KylinConfig config) {
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(config).getCubeDesc(descName);
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", descName, name);
@@ -144,9 +144,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 if (cubeDesc.isBroken()) {
 setStatus(RealizationStatusEnum.DESCBROKEN);
 logger.error("cube descriptor {} (for cube '{}') is broken", 
cubeDesc.getResourcePath(), name);
-for (String error : cubeDesc.getError()) {
-logger.error("Error: {}", error);
-}
+logger.error("Errors: {}", cubeDesc.getErrorsAsString());
 } else if (getStatus() == RealizationStatusEnum.DESCBROKEN) {
 setStatus(RealizationStatusEnum.DISABLED);
 logger.info("cube {} changed from DESCBROKEN to DISABLED", name);
@@ -217,7 +215,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() 
== RealizationStatusEnum.DESCBROKEN)
 && segments.isEmpty();
 }
-
+
 @Override
 public String resourceName() {
 return name;
@@ -301,7 +299,12 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 public int getCost() {
 int countedDimensionNum =

[22/50] [abbrv] kylin git commit: Minor, fix model draft not listed

2018-02-03 Thread liyang

Minor, fix model draft not listed


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/67f7197b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/67f7197b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/67f7197b

Branch: refs/heads/sync
Commit: 67f7197b849249df65d79bf3ec3ca090c0bb50a9
Parents: cc130e9
Author: nichunen 
Authored: Fri Jan 5 17:05:41 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../apache/kylin/rest/service/ModelService.java   |  2 +-
 .../kylin/rest/service/ModelServiceTest.java  | 18 ++
 2 files changed, 19 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/67f7197b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 43ce0b1..ee01f3e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -413,7 +413,7 @@ public class ModelService extends BasicService {
 RootPersistentEntity e = d.getEntity();
 if (e instanceof DataModelDesc) {
 DataModelDesc m = (DataModelDesc) e;
-if (modelName == null || modelName.equals(m.getName()))
+if (StringUtils.isEmpty(modelName) || 
modelName.equals(m.getName()))
 result.add(d);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/67f7197b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java 
b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index fff4ed9..aeb6d79 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -27,6 +27,8 @@ import java.util.List;
 
 import org.apache.kylin.common.persistence.Serializer;
 import org.apache.kylin.job.exception.JobException;
+import org.apache.kylin.metadata.draft.Draft;
+import org.apache.kylin.metadata.draft.DraftManager;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.ModelDimensionDesc;
 import org.junit.Assert;
@@ -132,6 +134,22 @@ public class ModelServiceTest extends ServiceTestBase {
 }
 }
 
+@Test
+public void testModelDraft() throws IOException {
+DraftManager mgr = DraftManager.getInstance(getTestConfig());
+// Create a draft of model
+Draft d = new Draft();
+d.setProject("default");
+d.updateRandomUuid();
+DataModelDesc modelDesc = modelService.getModel("ci_left_join_model", 
"default");
+d.setEntity(modelDesc);
+mgr.save(d);
+
+// Check list draft
+List draftList = modelService.listModelDrafts("", "default");
+Assert.assertEquals(draftList.size(), 1);
+}
+
 
 private String[] cutItems(String[] origin, int count) {
 if (origin == null)

kylin git commit: minor, add Spark into index.md

2018-01-29 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/document 42976f5c9 -> fd7ed711f


minor, add Spark into index.md


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fd7ed711
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fd7ed711
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fd7ed711

Branch: refs/heads/document
Commit: fd7ed711f320aa608e2b75273958b6dd801f7817
Parents: 42976f5
Author: Billy Liu 
Authored: Tue Jan 30 10:53:40 2018 +0800
Committer: Billy Liu 
Committed: Tue Jan 30 10:53:40 2018 +0800

--
 website/index.cn.md | 10 +-
 website/index.md| 12 ++--
 2 files changed, 11 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fd7ed711/website/index.cn.md
--
diff --git a/website/index.cn.md b/website/index.cn.md
index 39cffac..9978502 100644
--- a/website/index.cn.md
+++ b/website/index.cn.md
@@ -13,7 +13,7 @@ title: é¦é¡µ
   Apache Kylinâ¢ æ¦è§
   
   
-   Apache 
Kylinâ¢æ¯ä¸ä¸ªå¼æºçåå¸å¼åæå¼æï¼æä¾Hadoopä¹ä¸çSQLæ¥è¯¢æ¥å£åå¤ç»´åæï¼OLAPï¼è½åä»¥æ¯æè¶
å¤§è§æ¨¡æ°æ®ï¼æåç±eBay Inc. 
å¼åå¹¶è´¡ç®è³å¼æºç¤¾åºãå®è½å¨äºç§åæ¥è¯¢å·¨å¤§çHiveè¡¨ã
+   Apache 
Kylinâ¢æ¯ä¸ä¸ªå¼æºçåå¸å¼åæå¼æï¼æä¾Hadoop/Sparkä¹ä¸çSQLæ¥è¯¢æ¥å£åå¤ç»´åæï¼OLAPï¼è½åä»¥æ¯æè¶
å¤§è§æ¨¡æ°æ®ï¼æåç±eBay Inc. 
å¼åå¹¶è´¡ç®è³å¼æºç¤¾åºãå®è½å¨äºç§åæ¥è¯¢å·¨å¤§çHiveè¡¨ã

   
 
@@ -34,7 +34,7 @@ title: é¦é¡µ
   
 
- å¯æ©å±è¶å¿«OLAPå¼æ: 
-  Kylinæ¯ä¸ºåå°å¨Hadoopä¸ç¾äº¿è§æ¨¡æ°æ®æ¥è¯¢å»¶è¿èè®¾è®¡
+  Kylinæ¯ä¸ºåå°å¨Hadoop/Sparkä¸ç¾äº¿è§æ¨¡æ°æ®æ¥è¯¢å»¶è¿èè®¾è®¡
   
- Hadoop ANSI SQL æ¥å£: 
   Kylinä¸ºHadoopæä¾æ 
åSQLæ¯æå¤§é¨åæ¥è¯¢åè½
@@ -46,7 +46,7 @@ title: é¦é¡µ
   ç¨æ·è½å¤å¨Kylinéä¸ºç¾äº¿ä»¥ä¸æ°æ®éå®ä¹æ°æ®æ¨¡åå¹¶æå»ºç«æ¹ä½
   
- ä¸BIå·¥å·æ ç¼æ´å:
-  Kylinæä¾ä¸BIå·¥å
·ï¼å¦Tableauï¼çæ´åè½åï¼å³å°æä¾å¯¹å¶ä»å·¥å·çæ´å
+  Kylinæä¾ä¸BIå·¥å
·çæ´åè½åï¼å¦Tableauï¼PowerBI/Excelï¼MSTRï¼QlikSenseï¼HueåSuperSet
   
- å¶ä»ç¹æ§: 
   - Jobç®¡çä¸çæ§ 
@@ -55,8 +55,8 @@ title: é¦é¡µ
 - å©ç¨HBase Coprocessor
 - åºäºHyperLogLogçDinstinc Countè¿ä¼¼ç®æ³ 
 - åå¥½çwebçé¢ä»¥ç®¡çï¼çæ§åä½¿ç¨ç«æ¹ä½ 
-- é¡¹ç®åç«æ¹ä½çº§å«çè®¿é®æ§å¶å®å¨
-- æ¯æLDAP 
+- é¡¹ç®åè¡¨çº§å«çè®¿é®æ§å¶å®å¨
+- æ¯æLDAPãSSO 
   
 
   

http://git-wip-us.apache.org/repos/asf/kylin/blob/fd7ed711/website/index.md
--
diff --git a/website/index.md b/website/index.md
index d87c04e..13ae973 100644
--- a/website/index.md
+++ b/website/index.md
@@ -12,11 +12,11 @@ title: Home
   Apache Kylinâ¢ Overview
   
   
-  Apache Kylinâ¢ is an open source 
Distributed Analytics Engine designed to provide SQL interface and 
multi-dimensional analysis (OLAP) on Hadoop supporting extremely large 
datasets, original contributed from eBay Inc.
+  Apache Kylinâ¢ is an open source 
Distributed Analytics Engine designed to provide SQL interface and 
multi-dimensional analysis (OLAP) on Hadoop/Spark supporting extremely large 
datasets, original contributed from eBay Inc.
   Apache Kylinâ¢ lets you query massive 
data set at sub-second latency in 3 steps.
   
 
-  Identify a Star Schema on Hadoop.
+  Identify a Star/Snowfalke Schema on Hadoop.
   Build Cube from the identified tables.
   Query with ANSI-SQL and get results in sub-second, via 
ODBC, JDBC or RESTful API.
 
@@ -41,10 +41,10 @@ title: Home
   
 
- Extremely Fast OLAP Engine at Scale: 
-  Kylin is designed to reduce query latency on 
Hadoop for 10+ billions of rows of data
+  Kylin is designed to reduce query latency on 
Hadoop/Spark for 10+ billions of rows of data
   
- ANSI SQL Interface on Hadoop: 
-  Kylin offers ANSI SQL on Hadoop and supports 
most ANSI SQL

[6/6] kylin git commit: minor, stablize org.apache.kylin.dict.TrieDictionaryForestTest#englishWordsTest

2018-01-28 Thread lidong

minor, stablize org.apache.kylin.dict.TrieDictionaryForestTest#englishWordsTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eedd135
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eedd135
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eedd135

Branch: refs/heads/master
Commit: 5eedd135d27da96c2242079fd9e188fce0518f90
Parents: 9198cfe
Author: lidongsjtu 
Authored: Sun Jan 28 19:55:07 2018 +0800
Committer: lidongsjtu 
Committed: Sun Jan 28 19:55:07 2018 +0800

--
 .../kylin/dict/TrieDictionaryForestTest.java| 28 ++--
 1 file changed, 20 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eedd135/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
 
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 099075c..1f63111 100644
--- 
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ 
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -43,6 +43,7 @@ import java.util.NoSuchElementException;
 import java.util.Random;
 import java.util.TreeSet;
 
+import org.apache.kylin.common.util.Bytes;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -187,8 +188,14 @@ public class TrieDictionaryForestTest {
 str.add("party");
 str.add("parties");
 str.add("paint");
-String longStr = 
"paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
-+ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
 + 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
+String longStr = 
"paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
++ 
"dkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
 System.out.println("The length of the long string is " + 
longStr.length());
 str.add(longStr);
 
@@ -740,7 +747,8 @@ public class TrieDictionaryForestTest {
 System.out.println("times:" + i);
 }
 
-System.out.println("compare build time.  Old trie : " + 
oldDictTotalBuildTime / 1000.0 + "s.New trie : " + newDictTotalBuildTime / 
1000.0 + "s");
+System.out.println("compare build time.  Old trie : " + 
oldDictTotalBuildTime / 1000.0 + "s.New trie : "
++ newDictTotalBuildTime / 1000.0 + "s");
 }
 
 private void

[5/6] kylin git commit: minor, fix some typos

2018-01-28 Thread lidong

minor, fix some typos


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9198cfe8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9198cfe8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9198cfe8

Branch: refs/heads/master
Commit: 9198cfe8f1439246eff6abff38b3869453325aca
Parents: 74e3f61
Author: lidongsjtu 
Authored: Sun Jan 28 19:43:00 2018 +0800
Committer: lidongsjtu 
Committed: Sun Jan 28 19:43:00 2018 +0800

--
 .../java/org/apache/kylin/source/hive/HiveMetadataExplorer.java  | 4 ++--
 .../src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9198cfe8/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index cb3eb02..9d4cc53 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -185,7 +185,7 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 @Override
 public ColumnDesc[] evalQueryMetadata(String query) {
 if (StringUtils.isEmpty(query)) {
-throw new RuntimeException("Evalutate query shall not be empty.");
+throw new RuntimeException("Evaluate query shall not be empty.");
 }
 
 KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -204,7 +204,7 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 HiveTableMeta hiveTableMeta = 
hiveClient.getHiveTableMeta(tmpDatabase, tmpView);
 return extractColumnFromMeta(hiveTableMeta);
 } catch (Exception e) {
-throw new RuntimeException("Cannot evalutate metadata of query: " 
+ query, e);
+throw new RuntimeException("Cannot evaluate metadata of query: " + 
query, e);
 } finally {
 try {
 logger.debug("Cleaning up.");

http://git-wip-us.apache.org/repos/asf/kylin/blob/9198cfe8/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java 
b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index d96a68d..c34597a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -232,7 +232,7 @@ public class JdbcExplorer implements 
ISourceMetadataExplorer, ISampleDataDeploye
 @Override
 public ColumnDesc[] evalQueryMetadata(String query) {
 if (StringUtils.isEmpty(query)) {
-throw new RuntimeException("Evalutate query shall not be empty.");
+throw new RuntimeException("Evaluate query shall not be empty.");
 }
 
 KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -256,7 +256,7 @@ public class JdbcExplorer implements 
ISourceMetadataExplorer, ISampleDataDeploye
 DBUtils.closeQuietly(con);
 return result;
 } catch (Exception e) {
-throw new RuntimeException("Cannot evalutate metadata of query: " 
+ query, e);
+throw new RuntimeException("Cannot evaluate metadata of query: " + 
query, e);
 } finally {
 try {
 logger.debug("Cleaning up.");

[26/34] kylin git commit: minor, create job work dir with Materialize Hive View in Lookup Tables job.

2018-01-26 Thread liyang

minor, create job work dir with Materialize Hive View in Lookup Tables job.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0ae39b5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0ae39b5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0ae39b5b

Branch: refs/heads/master
Commit: 0ae39b5bb1cb05d3660ffb94c125cd8e85b91902
Parents: 7d5fb85
Author: yiming.xu <100650...@qq.com>
Authored: Tue Jan 16 14:04:10 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../org/apache/kylin/source/hive/HiveMRInput.java   | 16 
 1 file changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0ae39b5b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index e4564d0..6f42961 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -210,6 +210,9 @@ public class HiveMRInput implements IMRInput {
 if (lookupViewsTables.size() == 0) {
 return null;
 }
+// Create work dir to avoid hive create it,
+// the difference is that the owners are different.
+checkAndCreateWorkDir(jobWorkingDir);
 
 HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
 
hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
@@ -239,6 +242,19 @@ public class HiveMRInput implements IMRInput {
 return step;
 }
 
+private void checkAndCreateWorkDir(String jobWorkingDir) {
+try {
+Path path = new Path(jobWorkingDir);
+FileSystem fileSystem = HadoopUtil.getFileSystem(path);
+if (!fileSystem.exists(path)) {
+logger.info("Create jobWorkDir : " + jobWorkingDir);
+fileSystem.mkdirs(path);
+}
+} catch (IOException e) {
+logger.error("Could not create lookUp table dir : " + 
jobWorkingDir);
+}
+}
+
 private AbstractExecutable createFlatHiveTableStep(String 
hiveInitStatements, String jobWorkingDir,
 String cubeName) {
 //from hive to hive

[17/34] kylin git commit: minor, refine basic tuple filter class

2018-01-26 Thread liyang

minor, refine basic tuple filter class


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2b69155
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2b69155
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2b69155

Branch: refs/heads/master
Commit: c2b69155215f72a5f78bf65995390431ff424c1e
Parents: 1f33dd9
Author: Roger Shi 
Authored: Mon Jan 8 17:57:57 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../metadata/filter/LogicalTupleFilter.java | 26 
 .../kylin/metadata/filter/TupleFilter.java  | 43 +
 .../kylin/metadata/filter/TupleFilterTest.java  | 65 
 3 files changed, 134 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2b69155/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 7893ed8..f0c825f 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -170,4 +170,30 @@ public class LogicalTupleFilter extends TupleFilter 
implements IOptimizeableTupl
 return transformer.visit(this);
 }
 
+@Override
+public boolean equals(Object other) {
+if (other == this) {
+return true;
+}
+if (!(other instanceof LogicalTupleFilter)) {
+return false;
+}
+final LogicalTupleFilter otherFilter = (LogicalTupleFilter) other;
+if (otherFilter.operator != this.operator || 
otherFilter.children.size() != this.children.size()) {
+return false;
+}
+
+for (int i = 0; i < otherFilter.children.size(); i++) {
+if (!otherFilter.children.get(i).equals(this.children.get(i))) {
+return false;
+}
+}
+
+return true;
+}
+
+@Override
+public int hashCode() {
+return (operator == null ? 0 : operator.hashCode()) + 31 * 
this.children.hashCode();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2b69155/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 5ba8726..09b41f5 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -99,6 +99,11 @@ public abstract class TupleFilter {
 addChild(c); // subclass overrides addChild()
 }
 
+final public void addChildren(TupleFilter... children) {
+for (TupleFilter c : children)
+addChild(c); // subclass overrides addChild()
+}
+
 public List getChildren() {
 return children;
 }
@@ -121,6 +126,44 @@ public abstract class TupleFilter {
 }
 
 /**
+ * The storage level dislike NOT logic
+ */
+public TupleFilter removeNot() {
+return removeNotInternal(this);
+}
+
+private TupleFilter removeNotInternal(TupleFilter filter) {
+FilterOperatorEnum op = filter.getOperator();
+
+if (!(filter instanceof LogicalTupleFilter)) {
+return filter;
+}
+
+LogicalTupleFilter logicalFilter = (LogicalTupleFilter) filter;
+
+switch (logicalFilter.operator) {
+case NOT:
+assert (filter.children.size() == 1);
+TupleFilter reverse = filter.children.get(0).reverse();
+return removeNotInternal(reverse);
+case AND:
+LogicalTupleFilter andFilter = new 
LogicalTupleFilter(FilterOperatorEnum.AND);
+for (TupleFilter child : logicalFilter.children) {
+andFilter.addChild(removeNotInternal(child));
+}
+return andFilter;
+case OR:
+LogicalTupleFilter orFilter = new 
LogicalTupleFilter(FilterOperatorEnum.OR);
+for (TupleFilter child : logicalFilter.children) {
+orFilter.addChild(removeNotInternal(child));
+}
+return orFilter;
+default:
+throw new IllegalStateException("This filter is unexpected: " + 
filter);
+}
+}
+
+/**
  * flatten to OR-AND filter, (A AND B AND ..) OR (C AND D AND ..) OR

[32/34] kylin git commit: minor refactor: avoid exposing cubedesc's getError public api

2018-01-26 Thread liyang

minor refactor: avoid exposing cubedesc's getError public api


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59d50646
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59d50646
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59d50646

Branch: refs/heads/master
Commit: 59d5064673282ba36783e0796523ad83c2dc5e8c
Parents: 16236ef
Author: Hongbin Ma 
Authored: Fri Jan 26 19:13:12 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../org/apache/kylin/cube/CubeDescManager.java  |  2 +-
 .../org/apache/kylin/cube/CubeInstance.java | 29 ++
 .../org/apache/kylin/cube/model/CubeDesc.java   | 13 ++--
 .../kylin/rest/controller/CubeController.java   | 32 
 .../apache/kylin/rest/service/CubeService.java  | 12 
 5 files changed, 47 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59d50646/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index cd43f15..1db66e1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -198,7 +198,7 @@ public class CubeDescManager {
 
 postProcessCubeDesc(cubeDesc);
 // Check base validation
-if (!cubeDesc.getError().isEmpty()) {
+if (cubeDesc.isBroken()) {
 return cubeDesc;
 }
 // Semantic validation

http://git-wip-us.apache.org/repos/asf/kylin/blob/59d50646/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index a299203..6395abd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -6,15 +6,15 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
-*/
+ */
 
 package org.apache.kylin.cube;
 
@@ -130,13 +130,13 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // default constructor for jackson
 public CubeInstance() {
 }
-
+
 public CubeInstance latestCopyForWrite() {
 CubeManager mgr = CubeManager.getInstance(config);
 CubeInstance latest = mgr.getCube(name); // in case this object is 
out-of-date
 return mgr.copyForWrite(latest);
 }
-
+
 void init(KylinConfig config) {
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(config).getCubeDesc(descName);
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", descName, name);
@@ -144,9 +144,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 if (cubeDesc.isBroken()) {
 setStatus(RealizationStatusEnum.DESCBROKEN);
 logger.error("cube descriptor {} (for cube '{}') is broken", 
cubeDesc.getResourcePath(), name);
-for (String error : cubeDesc.getError()) {
-logger.error("Error: {}", error);
-}
+logger.error("Errors: {}", cubeDesc.getErrorsAsString());
 } else if (getStatus() == RealizationStatusEnum.DESCBROKEN) {
 setStatus(RealizationStatusEnum.DISABLED);
 logger.info("cube {} changed from DESCBROKEN to DISABLED", name);
@@ -217,7 +215,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() 
== RealizationStatusEnum.DESCBROKEN)
 && segments.isEmpty();
 }
-
+
 @Override
 public String resourceName() {
 return name;
@@ -301,7 +299,12 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 public int getCost() {
 int countedDimensionNum =

[27/34] kylin git commit: minor refactor

2018-01-26 Thread liyang

minor refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0988c846
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0988c846
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0988c846

Branch: refs/heads/master
Commit: 0988c846823ca65a46eb6ab9b3dcee443da57872
Parents: 0ae39b5
Author: Hongbin Ma 
Authored: Thu Jan 11 16:47:37 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../java/org/apache/kylin/rest/request/ModelRequest.java  | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0988c846/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java 
b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
index 5f07997..2c5b3bf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
@@ -26,6 +26,8 @@ public class ModelRequest {
 private boolean successful;
 private String message;
 private String project;
+
+private String ccInCheck;
 private boolean isSeekingExprAdvice;
 
 public String getUuid() {
@@ -112,4 +114,12 @@ public class ModelRequest {
 public void setIsSeekingExprAdvice(boolean seekingExprAdvice) {
 isSeekingExprAdvice = seekingExprAdvice;
 }
+
+public String getCcInCheck() {
+return ccInCheck;
+}
+
+public void setCcInCheck(String ccInCheck) {
+this.ccInCheck = ccInCheck;
+}
 }

[06/34] kylin git commit: Minor, get metadata of query columns

2018-01-26 Thread liyang

Minor, get metadata of query columns


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/25bb46e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/25bb46e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/25bb46e0

Branch: refs/heads/master
Commit: 25bb46e0cea270cae2c5b042a5766880fc2eea02
Parents: d30d2a3
Author: Yifan Zhang 
Authored: Thu Dec 28 21:02:16 2017 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../kylin/source/ISourceMetadataExplorer.java   |  3 +
 .../kylin/source/hive/HiveMetadataExplorer.java | 74 -
 .../apache/kylin/source/jdbc/JdbcExplorer.java  | 83 ++--
 .../apache/kylin/source/kafka/KafkaSource.java  |  6 ++
 4 files changed, 125 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/25bb46e0/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
 
b/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
index 4072846..0a6747c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
@@ -21,6 +21,7 @@ package org.apache.kylin.source;
 import java.util.List;
 
 import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableExtDesc;
 
@@ -33,4 +34,6 @@ public interface ISourceMetadataExplorer {
 Pair loadTableMetadata(String database, String 
table, String prj) throws Exception;
 
 List getRelatedKylinResources(TableDesc table);
+
+ColumnDesc[] evalQueryMetadata(String query);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/25bb46e0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index 55e6267..d80c546 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.metadata.TableMetadataManager;
@@ -59,7 +60,7 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 }
 
 TableDesc tableDesc = metaMgr.getTableDesc(database + "." + tableName, 
prj);
-
+
 // make a new TableDesc instance, don't modify the one in use
 if (tableDesc == null) {
 tableDesc = new TableDesc();
@@ -70,28 +71,12 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 } else {
 tableDesc = new TableDesc(tableDesc);
 }
-
+
 if (hiveTableMeta.tableType != null) {
 tableDesc.setTableType(hiveTableMeta.tableType);
 }
 
-int columnNumber = hiveTableMeta.allColumns.size();
-List columns = new ArrayList(columnNumber);
-for (int i = 0; i < columnNumber; i++) {
-HiveTableMeta.HiveTableColumnMeta field = 
hiveTableMeta.allColumns.get(i);
-ColumnDesc cdesc = new ColumnDesc();
-cdesc.setName(field.name.toUpperCase());
-// use "double" in kylin for "float"
-if ("float".equalsIgnoreCase(field.dataType)) {
-cdesc.setDatatype("double");
-} else {
-cdesc.setDatatype(field.dataType);
-}
-cdesc.setId(String.valueOf(i + 1));
-cdesc.setComment(field.comment);
-columns.add(cdesc);
-}
-tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
+tableDesc.setColumns(extractColumnFromMeta(hiveTableMeta));
 
 StringBuffer partitionColumnString = new StringBuffer();
 for (int i = 0, n = hiveTableMeta.partitionColumns.size(); i < n; i++) 
{
@@ -193,4 +178,55 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 return hiveDataType.toLowerCase();
 }
 
+@Override
+public ColumnDesc[]

[10/34] kylin git commit: minor, add array type test

2018-01-26 Thread liyang

minor, add array type test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ef3d490
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ef3d490
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ef3d490

Branch: refs/heads/master
Commit: 2ef3d4906937ad1b00bc344f8cfcbb8900ce0998
Parents: a4acb4d
Author: Roger Shi 
Authored: Thu Jan 4 16:52:23 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../apache/kylin/query/schema/OLAPTable.java|  2 +-
 .../kylin/query/schema/OLAPTableTest.java   | 78 
 2 files changed, 79 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2ef3d490/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java 
b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index ca8807e..b5313ed 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -88,7 +88,7 @@ public class OLAPTable extends AbstractQueryableTable 
implements TranslatableTab
 SQLTYPE_MAPPING.put("timestamp", SqlTypeName.TIMESTAMP);
 SQLTYPE_MAPPING.put("any", SqlTypeName.ANY);
 
-REGEX_SQLTYPE_MAPPING.put("array\\<.*\\>", SqlTypeName.ARRAY);
+REGEX_SQLTYPE_MAPPING.put("array\\<.+\\>", SqlTypeName.ARRAY);
 }
 
 private final boolean exposeMore;

http://git-wip-us.apache.org/repos/asf/kylin/blob/2ef3d490/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
--
diff --git 
a/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java 
b/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
new file mode 100644
index 000..00440de
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.schema;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.type.ArraySqlType;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class OLAPTableTest extends LocalFileMetadataTestCase {
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
+
+@Test
+public void testCreateSqlType() {
+RelDataTypeFactory typeFactory = new 
SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
+DataType kylinDataType = DataType.getType("array");
+
+RelDataType relDataType = OLAPTable.createSqlType(typeFactory, 
kylinDataType, true);
+Assert.assertTrue(relDataType instanceof ArraySqlType);
+Assert.assertEquals(SqlTypeName.ARRAY, relDataType.getSqlTypeName());
+Assert.assertTrue(relDataType.getComponentType() instanceof 
BasicSqlType);
+Assert.assertEquals(SqlTypeName.VARCHAR, 
relDataType.getComponentType().getSqlTypeName());
+Assert.assertTrue(relDataType.isNullable());
+
+kylinDataType = DataType.getType("array<>");
+boolean catchedEx = false;
+try {
+OLAPTable.createSqlType(typeFactory, kylinDataType, true);
+} catch (IllegalArgumentException ex) {
+Assert.assertEquals("Unrecognized data type

[08/34] kylin git commit: minor, restore OLAPToEnumerableConverter to original

2018-01-26 Thread liyang

minor, restore OLAPToEnumerableConverter to original


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a4acb4d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a4acb4d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a4acb4d5

Branch: refs/heads/master
Commit: a4acb4d53ef74f8e59f46f550143c599c546ff30
Parents: f73219c
Author: Roger Shi 
Authored: Wed Jan 3 22:21:28 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../apache/calcite/plan/OLAPRelMdRowCount.java  | 174 
 .../calcite/plan/OLAPRelMetadataProvider.java   |  39 
 .../org/apache/calcite/plan/RelOptCluster.java  | 199 ---
 .../src/test/resources/query/sql/query110.sql   |  58 ++
 .../relnode/OLAPToEnumerableConverter.java  |   2 +-
 5 files changed, 59 insertions(+), 413 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a4acb4d5/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java 
b/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
deleted file mode 100644
index 751561f..000
--- a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
+++ /dev/null
@@ -1,174 +0,0 @@
-package org.apache.calcite.plan;
-
-import org.apache.calcite.adapter.enumerable.EnumerableLimit;
-import org.apache.calcite.plan.volcano.RelSubset;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.SingleRel;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.Calc;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Intersect;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.Minus;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.SemiJoin;
-import org.apache.calcite.rel.core.Sort;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.core.Union;
-import org.apache.calcite.rel.core.Values;
-import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMdRowCount;
-import org.apache.calcite.rel.metadata.RelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.util.BuiltInMethod;
-
-public class OLAPRelMdRowCount extends RelMdRowCount {
-public static final RelMetadataProvider SOURCE = 
ReflectiveRelMetadataProvider
-.reflectiveSource(BuiltInMethod.ROW_COUNT.method, new 
OLAPRelMdRowCount());
-
-//avoid case like sql/query60.sql, which will generate plan:
-/*
-EnumerableLimit(fetch=[3])
-OLAPToEnumerableConverter
-OLAPAggregateRel(group=[{0}], SUM_PRICE=[SUM($1)], 
CNT_1=[COUNT()], TOTAL_ITEMS=[SUM($2)])
- */
-private boolean shouldIntercept(RelNode rel) {
-for (RelNode input : rel.getInputs()) {
-if (input instanceof RelSubset) {
-RelSubset relSubset = (RelSubset) input;
-if (relSubset.getBest() != null
-&& 
relSubset.getBest().getClass().getCanonicalName().endsWith("OLAPToEnumerableConverter"))
 {
-return true;
-}
-}
-}
-return false;
-}
-
-@Override
-public Double getRowCount(RelNode rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(RelSubset subset, RelMetadataQuery mq) {
-if (shouldIntercept(subset))
-return 1E10;
-
-return super.getRowCount(subset, mq);
-}
-
-@Override
-public Double getRowCount(Union rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Intersect rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Minus rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Filter rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Calc rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-

[02/34] kylin git commit: minor, add switch table-dir-create-first for build cube with hive step.

2018-01-26 Thread liyang

minor, add switch table-dir-create-first for build cube with hive step.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce1775a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce1775a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce1775a5

Branch: refs/heads/master
Commit: ce1775a5ab0e775bce6685e7801f114a4aff9b9b
Parents: 543124d
Author: yiming.xu <100650...@qq.com>
Authored: Wed Jan 24 17:33:18 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java   | 4 
 .../main/java/org/apache/kylin/source/hive/HiveMRInput.java  | 8 +---
 .../java/org/apache/kylin/source/hive/HiveMRInputTest.java   | 2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5045b2f..6d1b7f9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -755,6 +755,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.source.hive.sparksql-beeline-params", "");
 }
 
+public boolean getHiveTableDirCreateFirst() {
+return 
Boolean.parseBoolean(getOptional("kylin.source.hive.table-dir-create-first", 
"false"));
+}
+
 public String getFlatHiveTableClusterByDictColumn() {
 return 
getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 31a99cd..f02d2ea 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -179,9 +179,11 @@ public class HiveMRInput implements IMRInput {
 protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
 
 String jobWorkingDir = 
JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
-// Create work dir to avoid hive create it,
-// the difference is that the owners are different.
-checkAndCreateWorkDir(jobWorkingDir);
+if (KylinConfig.getInstanceFromEnv().getHiveTableDirCreateFirst()) 
{
+// Create work dir to avoid hive create it,
+// the difference is that the owners are different.
+checkAndCreateWorkDir(jobWorkingDir);
+}
 return jobWorkingDir;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
--
diff --git 
a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java 
b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index 6d0737d..f33ee74 100644
--- 
a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ 
b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -41,8 +41,8 @@ public class HiveMRInputTest {
 try {
 KylinConfig kylinConfig = mock(KylinConfig.class);
 KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true);
 
when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
-
 DefaultChainedExecutable defaultChainedExecutable = 
mock(DefaultChainedExecutable.class);
 defaultChainedExecutable.setId(UUID.randomUUID().toString());

[13/34] kylin git commit: Minor, fix model draft not listed

2018-01-26 Thread liyang

Minor, fix model draft not listed


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/67f7197b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/67f7197b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/67f7197b

Branch: refs/heads/master
Commit: 67f7197b849249df65d79bf3ec3ca090c0bb50a9
Parents: cc130e9
Author: nichunen 
Authored: Fri Jan 5 17:05:41 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../apache/kylin/rest/service/ModelService.java   |  2 +-
 .../kylin/rest/service/ModelServiceTest.java  | 18 ++
 2 files changed, 19 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/67f7197b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 43ce0b1..ee01f3e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -413,7 +413,7 @@ public class ModelService extends BasicService {
 RootPersistentEntity e = d.getEntity();
 if (e instanceof DataModelDesc) {
 DataModelDesc m = (DataModelDesc) e;
-if (modelName == null || modelName.equals(m.getName()))
+if (StringUtils.isEmpty(modelName) || 
modelName.equals(m.getName()))
 result.add(d);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/67f7197b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java 
b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index fff4ed9..aeb6d79 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -27,6 +27,8 @@ import java.util.List;
 
 import org.apache.kylin.common.persistence.Serializer;
 import org.apache.kylin.job.exception.JobException;
+import org.apache.kylin.metadata.draft.Draft;
+import org.apache.kylin.metadata.draft.DraftManager;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.ModelDimensionDesc;
 import org.junit.Assert;
@@ -132,6 +134,22 @@ public class ModelServiceTest extends ServiceTestBase {
 }
 }
 
+@Test
+public void testModelDraft() throws IOException {
+DraftManager mgr = DraftManager.getInstance(getTestConfig());
+// Create a draft of model
+Draft d = new Draft();
+d.setProject("default");
+d.updateRandomUuid();
+DataModelDesc modelDesc = modelService.getModel("ci_left_join_model", 
"default");
+d.setEntity(modelDesc);
+mgr.save(d);
+
+// Check list draft
+List draftList = modelService.listModelDrafts("", "default");
+Assert.assertEquals(draftList.size(), 1);
+}
+
 
 private String[] cutItems(String[] origin, int count) {
 if (origin == null)

[25/34] kylin git commit: minor, refine sql comment removing

2018-01-26 Thread liyang

minor, refine sql comment removing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7d5fb855
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7d5fb855
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7d5fb855

Branch: refs/heads/master
Commit: 7d5fb855064e2b81cd3b154cdeeafec4e64f63c9
Parents: 60caf61
Author: Roger Shi 
Authored: Tue Jan 16 15:17:46 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 22:54:58 2018 +0800

--
 .../org/apache/kylin/query/util/QueryUtil.java | 17 -
 .../org/apache/kylin/query/util/QueryUtilTest.java | 16 +++-
 2 files changed, 19 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7d5fb855/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java 
b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 22658f1..da18bd7 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -142,23 +142,14 @@ public class QueryUtil {
 
 public static String removeCommentInSql(String sql1) {
 // match two patterns, one is "-- comment", the other is "/* comment 
*/"
-final String[] commentPatterns = new String[] { "--[^\r\n]*", 
"/\\*[\\s\\S]*?\\*/" };
+final String[] commentPatterns = new String[] { "--.*?[\r\n]", 
"/\\*.*?\\*/" };
 
 for (int i = 0; i < commentPatterns.length; i++) {
-String commentPattern = commentPatterns[i];
-Pattern pattern = Pattern.compile(commentPattern);
-Matcher matcher = pattern.matcher(sql1);
-
-while (matcher.find()) {
-if (matcher.start() == 0) {
-sql1 = sql1.substring(matcher.end()).trim();
-} else {
-sql1 = (sql1.substring(0, matcher.start()) + 
sql1.substring(matcher.end())).trim();
-}
-matcher = pattern.matcher(sql1);
-}
+sql1 = sql1.replaceAll(commentPatterns[i], "");
 }
 
+sql1 = sql1.trim();
+
 return sql1;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7d5fb855/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
--
diff --git a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java 
b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index e7a7ecb..e3cd819 100644
--- a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -149,9 +149,23 @@ public class QueryUtilTest extends 
LocalFileMetadataTestCase {
 }
 
 {
-String sqlWithComment = "/* comment \n select count(*) from 
kylin_sales; \n */ " + originSql;
+String sqlWithComment = "/* comment1/comment2 */ " + originSql;
 Assert.assertEquals(originSql, 
QueryUtil.removeCommentInSql(sqlWithComment));
 }
 
+{
+String sqlWithComment = "/* comment1 * comment2 */ " + originSql;
+Assert.assertEquals(originSql, 
QueryUtil.removeCommentInSql(sqlWithComment));
+}
+
+{
+String sqlWithComment = "/* comment1 * comment2 */ /* comment3 / 
comment4 */ -- comment 5\n" + originSql;
+Assert.assertEquals(originSql, 
QueryUtil.removeCommentInSql(sqlWithComment));
+}
+
+{
+String sqlWithComment = "/* comment1 * comment2 */ -- comment 5\n" 
+ originSql + "/* comment3 / comment4 */";
+Assert.assertEquals(originSql, 
QueryUtil.removeCommentInSql(sqlWithComment));
+}
 }
 }

[03/33] kylin git commit: Minor, fix NotClassFound issue in org.apache.hadoop.hive.ql.Driver

2018-01-26 Thread liyang

Minor, fix NotClassFound issue in org.apache.hadoop.hive.ql.Driver


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca8428d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca8428d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca8428d3

Branch: refs/heads/sync
Commit: ca8428d388cb3b0716efc92fb5cec729d636204f
Parents: b4daa8a
Author: Yifan Zhang 
Authored: Mon Jan 1 16:33:26 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 17:24:08 2018 +0800

--
 .../kylin/source/hive/BeelineHiveClient.java|  5 +--
 .../apache/kylin/source/hive/CLIHiveClient.java | 39 +++-
 .../kylin/source/hive/HiveMetadataExplorer.java | 29 ++-
 .../apache/kylin/source/hive/IHiveClient.java   |  6 +--
 .../apache/kylin/source/jdbc/JdbcExplorer.java  |  9 -
 5 files changed, 43 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index ee693c5..314402c 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -28,7 +28,6 @@ import java.sql.Statement;
 import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.kylin.common.util.DBUtils;
 
 import com.google.common.base.Preconditions;
@@ -117,12 +116,12 @@ public class BeelineHiveClient implements IHiveClient {
 }
 
 @Override
-public void executeHQL(String hql) throws CommandNeedRetryException, 
IOException {
+public void executeHQL(String hql) throws IOException {
 throw new UnsupportedOperationException();
 }
 
 @Override
-public void executeHQL(String[] hqls) throws CommandNeedRetryException, 
IOException {
+public void executeHQL(String[] hqls) throws IOException {
 throw new UnsupportedOperationException();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca8428d3/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index e8a93bd..bc9f17e 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -22,17 +22,15 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.common.StatsSetupConst;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HiveCmdBuilder;
+import org.apache.kylin.common.util.Pair;
 
 import com.google.common.collect.Lists;
 
@@ -43,7 +41,6 @@ import com.google.common.collect.Lists;
  */
 public class CLIHiveClient implements IHiveClient {
 protected HiveConf hiveConf = null;
-protected Driver driver = null;
 protected HiveMetaStoreClient metaStoreClient = null;
 
 public CLIHiveClient() {
@@ -52,22 +49,25 @@ public class CLIHiveClient implements IHiveClient {
 
 /**
  * only used by Deploy Util
+ * @throws IOException 
  */
 @Override
-public void executeHQL(String hql) throws CommandNeedRetryException, 
IOException {
-CommandProcessorResponse response = getDriver().run(hql);
-int retCode = response.getResponseCode();
-if (retCode != 0) {
-String err = response.getErrorMessage();
-throw new IOException("Failed to execute hql [" + hql + "], error 
message is: " + err);
+public void executeHQL(String hql) throws IOException {
+final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+hiveCmdBuilder.addStatement(hql);
+

[01/33] kylin git commit: Minor, get metadata of query columns

2018-01-26 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/sync [created] 923a9bd9f


Minor, get metadata of query columns


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4196ddd4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4196ddd4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4196ddd4

Branch: refs/heads/sync
Commit: 4196ddd42b65809cfcfba102596224ee1f40312f
Parents: 34cbd18
Author: Yifan Zhang 
Authored: Thu Dec 28 21:02:16 2017 +0800
Committer: Li Yang 
Committed: Fri Jan 26 16:49:34 2018 +0800

--
 .../kylin/source/ISourceMetadataExplorer.java   |  3 +
 .../kylin/source/hive/HiveMetadataExplorer.java | 74 -
 .../apache/kylin/source/jdbc/JdbcExplorer.java  | 83 ++--
 .../apache/kylin/source/kafka/KafkaSource.java  |  6 ++
 4 files changed, 125 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4196ddd4/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
 
b/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
index 4072846..0a6747c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/source/ISourceMetadataExplorer.java
@@ -21,6 +21,7 @@ package org.apache.kylin.source;
 import java.util.List;
 
 import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableExtDesc;
 
@@ -33,4 +34,6 @@ public interface ISourceMetadataExplorer {
 Pair loadTableMetadata(String database, String 
table, String prj) throws Exception;
 
 List getRelatedKylinResources(TableDesc table);
+
+ColumnDesc[] evalQueryMetadata(String query);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4196ddd4/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index 55e6267..d80c546 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.metadata.TableMetadataManager;
@@ -59,7 +60,7 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 }
 
 TableDesc tableDesc = metaMgr.getTableDesc(database + "." + tableName, 
prj);
-
+
 // make a new TableDesc instance, don't modify the one in use
 if (tableDesc == null) {
 tableDesc = new TableDesc();
@@ -70,28 +71,12 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 } else {
 tableDesc = new TableDesc(tableDesc);
 }
-
+
 if (hiveTableMeta.tableType != null) {
 tableDesc.setTableType(hiveTableMeta.tableType);
 }
 
-int columnNumber = hiveTableMeta.allColumns.size();
-List columns = new ArrayList(columnNumber);
-for (int i = 0; i < columnNumber; i++) {
-HiveTableMeta.HiveTableColumnMeta field = 
hiveTableMeta.allColumns.get(i);
-ColumnDesc cdesc = new ColumnDesc();
-cdesc.setName(field.name.toUpperCase());
-// use "double" in kylin for "float"
-if ("float".equalsIgnoreCase(field.dataType)) {
-cdesc.setDatatype("double");
-} else {
-cdesc.setDatatype(field.dataType);
-}
-cdesc.setId(String.valueOf(i + 1));
-cdesc.setComment(field.comment);
-columns.add(cdesc);
-}
-tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
+tableDesc.setColumns(extractColumnFromMeta(hiveTableMeta));
 
 StringBuffer partitionColumnString = new StringBuffer();
 for (int i = 0, n = hiveTableMeta.partitionColumns.size(); i < n; i++) 
{
@@ -193,4 +178,55 @@ public class HiveMetadataExplorer implements 
ISourceMetadataExplorer, ISampleDat
 return

[23/33] kylin git commit: minor, create job work dir with Materialize Hive View in Lookup Tables job.

2018-01-26 Thread liyang

minor, create job work dir with Materialize Hive View in Lookup Tables job.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/de9b5286
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/de9b5286
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/de9b5286

Branch: refs/heads/sync
Commit: de9b528697729df97bffa26112c2e761e4cd5634
Parents: 327bd76
Author: yiming.xu <100650...@qq.com>
Authored: Tue Jan 16 14:04:10 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 19:01:08 2018 +0800

--
 .../org/apache/kylin/source/hive/HiveMRInput.java   | 16 
 1 file changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/de9b5286/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index e4564d0..6f42961 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -210,6 +210,9 @@ public class HiveMRInput implements IMRInput {
 if (lookupViewsTables.size() == 0) {
 return null;
 }
+// Create work dir to avoid hive create it,
+// the difference is that the owners are different.
+checkAndCreateWorkDir(jobWorkingDir);
 
 HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
 
hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
@@ -239,6 +242,19 @@ public class HiveMRInput implements IMRInput {
 return step;
 }
 
+private void checkAndCreateWorkDir(String jobWorkingDir) {
+try {
+Path path = new Path(jobWorkingDir);
+FileSystem fileSystem = HadoopUtil.getFileSystem(path);
+if (!fileSystem.exists(path)) {
+logger.info("Create jobWorkDir : " + jobWorkingDir);
+fileSystem.mkdirs(path);
+}
+} catch (IOException e) {
+logger.error("Could not create lookUp table dir : " + 
jobWorkingDir);
+}
+}
+
 private AbstractExecutable createFlatHiveTableStep(String 
hiveInitStatements, String jobWorkingDir,
 String cubeName) {
 //from hive to hive

[05/33] kylin git commit: minor, restore OLAPToEnumerableConverter to original

2018-01-26 Thread liyang

minor, restore OLAPToEnumerableConverter to original


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd048b96
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd048b96
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd048b96

Branch: refs/heads/sync
Commit: bd048b96971ed92e5e5dc1cc05173bd678ae3c24
Parents: 91ed19b
Author: Roger Shi 
Authored: Wed Jan 3 22:21:28 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 17:30:46 2018 +0800

--
 .../apache/calcite/plan/OLAPRelMdRowCount.java  | 174 
 .../calcite/plan/OLAPRelMetadataProvider.java   |  39 
 .../org/apache/calcite/plan/RelOptCluster.java  | 199 ---
 .../src/test/resources/query/sql/query110.sql   |  58 ++
 .../relnode/OLAPToEnumerableConverter.java  |   2 +-
 5 files changed, 59 insertions(+), 413 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bd048b96/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java 
b/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
deleted file mode 100644
index 751561f..000
--- a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
+++ /dev/null
@@ -1,174 +0,0 @@
-package org.apache.calcite.plan;
-
-import org.apache.calcite.adapter.enumerable.EnumerableLimit;
-import org.apache.calcite.plan.volcano.RelSubset;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.SingleRel;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.Calc;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Intersect;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.Minus;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.SemiJoin;
-import org.apache.calcite.rel.core.Sort;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.core.Union;
-import org.apache.calcite.rel.core.Values;
-import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMdRowCount;
-import org.apache.calcite.rel.metadata.RelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.util.BuiltInMethod;
-
-public class OLAPRelMdRowCount extends RelMdRowCount {
-public static final RelMetadataProvider SOURCE = 
ReflectiveRelMetadataProvider
-.reflectiveSource(BuiltInMethod.ROW_COUNT.method, new 
OLAPRelMdRowCount());
-
-//avoid case like sql/query60.sql, which will generate plan:
-/*
-EnumerableLimit(fetch=[3])
-OLAPToEnumerableConverter
-OLAPAggregateRel(group=[{0}], SUM_PRICE=[SUM($1)], 
CNT_1=[COUNT()], TOTAL_ITEMS=[SUM($2)])
- */
-private boolean shouldIntercept(RelNode rel) {
-for (RelNode input : rel.getInputs()) {
-if (input instanceof RelSubset) {
-RelSubset relSubset = (RelSubset) input;
-if (relSubset.getBest() != null
-&& 
relSubset.getBest().getClass().getCanonicalName().endsWith("OLAPToEnumerableConverter"))
 {
-return true;
-}
-}
-}
-return false;
-}
-
-@Override
-public Double getRowCount(RelNode rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(RelSubset subset, RelMetadataQuery mq) {
-if (shouldIntercept(subset))
-return 1E10;
-
-return super.getRowCount(subset, mq);
-}
-
-@Override
-public Double getRowCount(Union rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Intersect rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Minus rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Filter rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-return super.getRowCount(rel, mq);
-}
-
-@Override
-public Double getRowCount(Calc rel, RelMetadataQuery mq) {
-if (shouldIntercept(rel))
-return 1E10;
-
-

[13/33] kylin git commit: minor, refine basic tuple filter class

2018-01-26 Thread liyang

minor, refine basic tuple filter class


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d43abbd6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d43abbd6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d43abbd6

Branch: refs/heads/sync
Commit: d43abbd6bb722f85035d432ae05399e55e2917e8
Parents: 13f64f4
Author: Roger Shi 
Authored: Mon Jan 8 17:57:57 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 18:46:52 2018 +0800

--
 .../metadata/filter/LogicalTupleFilter.java | 26 
 .../kylin/metadata/filter/TupleFilter.java  | 43 +
 .../kylin/metadata/filter/TupleFilterTest.java  | 65 
 3 files changed, 134 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d43abbd6/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 7893ed8..f0c825f 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -170,4 +170,30 @@ public class LogicalTupleFilter extends TupleFilter 
implements IOptimizeableTupl
 return transformer.visit(this);
 }
 
+@Override
+public boolean equals(Object other) {
+if (other == this) {
+return true;
+}
+if (!(other instanceof LogicalTupleFilter)) {
+return false;
+}
+final LogicalTupleFilter otherFilter = (LogicalTupleFilter) other;
+if (otherFilter.operator != this.operator || 
otherFilter.children.size() != this.children.size()) {
+return false;
+}
+
+for (int i = 0; i < otherFilter.children.size(); i++) {
+if (!otherFilter.children.get(i).equals(this.children.get(i))) {
+return false;
+}
+}
+
+return true;
+}
+
+@Override
+public int hashCode() {
+return (operator == null ? 0 : operator.hashCode()) + 31 * 
this.children.hashCode();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d43abbd6/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 5ba8726..09b41f5 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -99,6 +99,11 @@ public abstract class TupleFilter {
 addChild(c); // subclass overrides addChild()
 }
 
+final public void addChildren(TupleFilter... children) {
+for (TupleFilter c : children)
+addChild(c); // subclass overrides addChild()
+}
+
 public List getChildren() {
 return children;
 }
@@ -121,6 +126,44 @@ public abstract class TupleFilter {
 }
 
 /**
+ * The storage level dislike NOT logic
+ */
+public TupleFilter removeNot() {
+return removeNotInternal(this);
+}
+
+private TupleFilter removeNotInternal(TupleFilter filter) {
+FilterOperatorEnum op = filter.getOperator();
+
+if (!(filter instanceof LogicalTupleFilter)) {
+return filter;
+}
+
+LogicalTupleFilter logicalFilter = (LogicalTupleFilter) filter;
+
+switch (logicalFilter.operator) {
+case NOT:
+assert (filter.children.size() == 1);
+TupleFilter reverse = filter.children.get(0).reverse();
+return removeNotInternal(reverse);
+case AND:
+LogicalTupleFilter andFilter = new 
LogicalTupleFilter(FilterOperatorEnum.AND);
+for (TupleFilter child : logicalFilter.children) {
+andFilter.addChild(removeNotInternal(child));
+}
+return andFilter;
+case OR:
+LogicalTupleFilter orFilter = new 
LogicalTupleFilter(FilterOperatorEnum.OR);
+for (TupleFilter child : logicalFilter.children) {
+orFilter.addChild(removeNotInternal(child));
+}
+return orFilter;
+default:
+throw new IllegalStateException("This filter is unexpected: " + 
filter);
+}
+}
+
+/**
  * flatten to OR-AND filter, (A AND B AND ..) OR (C AND D AND ..) OR ..

[30/33] kylin git commit: minor refactor: avoid exposing cubedesc's getError public api

2018-01-26 Thread liyang

minor refactor: avoid exposing cubedesc's getError public api


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2062d6e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2062d6e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2062d6e7

Branch: refs/heads/sync
Commit: 2062d6e74550102bebda7aac1abfa52afb1fa6de
Parents: e071c1c
Author: Hongbin Ma 
Authored: Fri Jan 26 19:13:12 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 19:13:12 2018 +0800

--
 .../org/apache/kylin/cube/CubeDescManager.java  |  2 +-
 .../org/apache/kylin/cube/CubeInstance.java | 29 ++
 .../org/apache/kylin/cube/model/CubeDesc.java   | 13 ++--
 .../kylin/rest/controller/CubeController.java   | 32 
 .../apache/kylin/rest/service/CubeService.java  | 12 
 5 files changed, 47 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2062d6e7/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index cd43f15..1db66e1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -198,7 +198,7 @@ public class CubeDescManager {
 
 postProcessCubeDesc(cubeDesc);
 // Check base validation
-if (!cubeDesc.getError().isEmpty()) {
+if (cubeDesc.isBroken()) {
 return cubeDesc;
 }
 // Semantic validation

http://git-wip-us.apache.org/repos/asf/kylin/blob/2062d6e7/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index a299203..6395abd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -6,15 +6,15 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
-*/
+ */
 
 package org.apache.kylin.cube;
 
@@ -130,13 +130,13 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // default constructor for jackson
 public CubeInstance() {
 }
-
+
 public CubeInstance latestCopyForWrite() {
 CubeManager mgr = CubeManager.getInstance(config);
 CubeInstance latest = mgr.getCube(name); // in case this object is 
out-of-date
 return mgr.copyForWrite(latest);
 }
-
+
 void init(KylinConfig config) {
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(config).getCubeDesc(descName);
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", descName, name);
@@ -144,9 +144,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 if (cubeDesc.isBroken()) {
 setStatus(RealizationStatusEnum.DESCBROKEN);
 logger.error("cube descriptor {} (for cube '{}') is broken", 
cubeDesc.getResourcePath(), name);
-for (String error : cubeDesc.getError()) {
-logger.error("Error: {}", error);
-}
+logger.error("Errors: {}", cubeDesc.getErrorsAsString());
 } else if (getStatus() == RealizationStatusEnum.DESCBROKEN) {
 setStatus(RealizationStatusEnum.DISABLED);
 logger.info("cube {} changed from DESCBROKEN to DISABLED", name);
@@ -217,7 +215,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() 
== RealizationStatusEnum.DESCBROKEN)
 && segments.isEmpty();
 }
-
+
 @Override
 public String resourceName() {
 return name;
@@ -301,7 +299,12 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 public int getCost() {
 int countedDimensionNum =

[22/33] kylin git commit: minor, refine sql comment removing

2018-01-26 Thread liyang

minor, refine sql comment removing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/327bd766
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/327bd766
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/327bd766

Branch: refs/heads/sync
Commit: 327bd766c85e2fd0b6a335b38f96fe3a9d753bdf
Parents: 5977985
Author: Roger Shi 
Authored: Tue Jan 16 15:17:46 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 19:00:23 2018 +0800

--
 .../org/apache/kylin/query/util/QueryUtil.java | 17 -
 .../org/apache/kylin/query/util/QueryUtilTest.java | 16 +++-
 2 files changed, 19 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/327bd766/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java 
b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 22658f1..da18bd7 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -142,23 +142,14 @@ public class QueryUtil {
 
 public static String removeCommentInSql(String sql1) {
 // match two patterns, one is "-- comment", the other is "/* comment 
*/"
-final String[] commentPatterns = new String[] { "--[^\r\n]*", 
"/\\*[\\s\\S]*?\\*/" };
+final String[] commentPatterns = new String[] { "--.*?[\r\n]", 
"/\\*.*?\\*/" };
 
 for (int i = 0; i < commentPatterns.length; i++) {
-String commentPattern = commentPatterns[i];
-Pattern pattern = Pattern.compile(commentPattern);
-Matcher matcher = pattern.matcher(sql1);
-
-while (matcher.find()) {
-if (matcher.start() == 0) {
-sql1 = sql1.substring(matcher.end()).trim();
-} else {
-sql1 = (sql1.substring(0, matcher.start()) + 
sql1.substring(matcher.end())).trim();
-}
-matcher = pattern.matcher(sql1);
-}
+sql1 = sql1.replaceAll(commentPatterns[i], "");
 }
 
+sql1 = sql1.trim();
+
 return sql1;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/327bd766/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
--
diff --git a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java 
b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index e7a7ecb..e3cd819 100644
--- a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -149,9 +149,23 @@ public class QueryUtilTest extends 
LocalFileMetadataTestCase {
 }
 
 {
-String sqlWithComment = "/* comment \n select count(*) from 
kylin_sales; \n */ " + originSql;
+String sqlWithComment = "/* comment1/comment2 */ " + originSql;
 Assert.assertEquals(originSql, 
QueryUtil.removeCommentInSql(sqlWithComment));
 }
 
+{
+String sqlWithComment = "/* comment1 * comment2 */ " + originSql;
+Assert.assertEquals(originSql, 
QueryUtil.removeCommentInSql(sqlWithComment));
+}
+
+{
+String sqlWithComment = "/* comment1 * comment2 */ /* comment3 / 
comment4 */ -- comment 5\n" + originSql;
+Assert.assertEquals(originSql, 
QueryUtil.removeCommentInSql(sqlWithComment));
+}
+
+{
+String sqlWithComment = "/* comment1 * comment2 */ -- comment 5\n" 
+ originSql + "/* comment3 / comment4 */";
+Assert.assertEquals(originSql, 
QueryUtil.removeCommentInSql(sqlWithComment));
+}
 }
 }

[06/33] kylin git commit: minor, add array type test

2018-01-26 Thread liyang

minor, add array type test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b10eb7cb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b10eb7cb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b10eb7cb

Branch: refs/heads/sync
Commit: b10eb7cbaf61355535d67d7fc542a72e39ad976f
Parents: bd048b9
Author: Roger Shi 
Authored: Thu Jan 4 16:52:23 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 17:31:36 2018 +0800

--
 .../apache/kylin/query/schema/OLAPTable.java|  2 +-
 .../kylin/query/schema/OLAPTableTest.java   | 78 
 2 files changed, 79 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b10eb7cb/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java 
b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index ca8807e..b5313ed 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -88,7 +88,7 @@ public class OLAPTable extends AbstractQueryableTable 
implements TranslatableTab
 SQLTYPE_MAPPING.put("timestamp", SqlTypeName.TIMESTAMP);
 SQLTYPE_MAPPING.put("any", SqlTypeName.ANY);
 
-REGEX_SQLTYPE_MAPPING.put("array\\<.*\\>", SqlTypeName.ARRAY);
+REGEX_SQLTYPE_MAPPING.put("array\\<.+\\>", SqlTypeName.ARRAY);
 }
 
 private final boolean exposeMore;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b10eb7cb/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
--
diff --git 
a/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java 
b/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
new file mode 100644
index 000..00440de
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.schema;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.type.ArraySqlType;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class OLAPTableTest extends LocalFileMetadataTestCase {
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
+
+@Test
+public void testCreateSqlType() {
+RelDataTypeFactory typeFactory = new 
SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
+DataType kylinDataType = DataType.getType("array");
+
+RelDataType relDataType = OLAPTable.createSqlType(typeFactory, 
kylinDataType, true);
+Assert.assertTrue(relDataType instanceof ArraySqlType);
+Assert.assertEquals(SqlTypeName.ARRAY, relDataType.getSqlTypeName());
+Assert.assertTrue(relDataType.getComponentType() instanceof 
BasicSqlType);
+Assert.assertEquals(SqlTypeName.VARCHAR, 
relDataType.getComponentType().getSqlTypeName());
+Assert.assertTrue(relDataType.isNullable());
+
+kylinDataType = DataType.getType("array<>");
+boolean catchedEx = false;
+try {
+OLAPTable.createSqlType(typeFactory, kylinDataType, true);
+} catch (IllegalArgumentException ex) {
+Assert.assertEquals("Unrecognized data type

[24/33] kylin git commit: minor refactor

2018-01-26 Thread liyang

minor refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5bdf7231
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5bdf7231
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5bdf7231

Branch: refs/heads/sync
Commit: 5bdf72314827dcb9d8282566691c5bf8f6278dee
Parents: de9b528
Author: Hongbin Ma 
Authored: Thu Jan 11 16:47:37 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 19:01:50 2018 +0800

--
 .../java/org/apache/kylin/rest/request/ModelRequest.java  | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5bdf7231/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java 
b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
index 5f07997..2c5b3bf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
@@ -26,6 +26,8 @@ public class ModelRequest {
 private boolean successful;
 private String message;
 private String project;
+
+private String ccInCheck;
 private boolean isSeekingExprAdvice;
 
 public String getUuid() {
@@ -112,4 +114,12 @@ public class ModelRequest {
 public void setIsSeekingExprAdvice(boolean seekingExprAdvice) {
 isSeekingExprAdvice = seekingExprAdvice;
 }
+
+public String getCcInCheck() {
+return ccInCheck;
+}
+
+public void setCcInCheck(String ccInCheck) {
+this.ccInCheck = ccInCheck;
+}
 }

[32/33] kylin git commit: minor, add switch table-dir-create-first for build cube with hive step.

2018-01-26 Thread liyang

minor, add switch table-dir-create-first for build cube with hive step.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ad6acbe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ad6acbe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ad6acbe

Branch: refs/heads/sync
Commit: 4ad6acbe85d33159cc85bf23739c3c9e8f22fb33
Parents: 4559101
Author: yiming.xu <100650...@qq.com>
Authored: Wed Jan 24 17:33:18 2018 +0800
Committer: Li Yang 
Committed: Fri Jan 26 19:15:49 2018 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java   | 4 
 .../main/java/org/apache/kylin/source/hive/HiveMRInput.java  | 8 +---
 .../java/org/apache/kylin/source/hive/HiveMRInputTest.java   | 2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4ad6acbe/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index edb5b29..b6d8c02 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -755,6 +755,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.source.hive.sparksql-beeline-params", "");
 }
 
+public boolean getHiveTableDirCreateFirst() {
+return 
Boolean.parseBoolean(getOptional("kylin.source.hive.table-dir-create-first", 
"false"));
+}
+
 public String getFlatHiveTableClusterByDictColumn() {
 return 
getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4ad6acbe/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 31a99cd..f02d2ea 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -179,9 +179,11 @@ public class HiveMRInput implements IMRInput {
 protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
 
 String jobWorkingDir = 
JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
-// Create work dir to avoid hive create it,
-// the difference is that the owners are different.
-checkAndCreateWorkDir(jobWorkingDir);
+if (KylinConfig.getInstanceFromEnv().getHiveTableDirCreateFirst()) 
{
+// Create work dir to avoid hive create it,
+// the difference is that the owners are different.
+checkAndCreateWorkDir(jobWorkingDir);
+}
 return jobWorkingDir;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/4ad6acbe/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
--
diff --git 
a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java 
b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index 6d0737d..f33ee74 100644
--- 
a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ 
b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -41,8 +41,8 @@ public class HiveMRInputTest {
 try {
 KylinConfig kylinConfig = mock(KylinConfig.class);
 KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true);
 
when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
-
 DefaultChainedExecutable defaultChainedExecutable = 
mock(DefaultChainedExecutable.class);
 defaultChainedExecutable.setId(UUID.randomUUID().toString());

kylin git commit: minor, add codacy badge

2018-01-26 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master b361844ce -> 34cbd1813


minor, add codacy badge


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/34cbd181
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/34cbd181
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/34cbd181

Branch: refs/heads/master
Commit: 34cbd18133362fa63207a2aa7a234cc3e82c5531
Parents: b361844
Author: lidongsjtu 
Authored: Fri Jan 26 16:14:31 2018 +0800
Committer: lidongsjtu 
Committed: Fri Jan 26 16:14:31 2018 +0800

--
 README.md | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/34cbd181/README.md
--
diff --git a/README.md b/README.md
index df8fd3a..dd66cb9 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,7 @@ Apache Kylin
 
 [![Build 
Status](https://travis-ci.org/apache/kylin.svg?branch=master)](https://travis-ci.org/apache/kylin)
 
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
+[![Codacy 
Badge](https://api.codacy.com/project/badge/Grade/74f0139786cd4e8a8ce69bb0c17c2e71)](https://www.codacy.com/app/kyligence-git/kylin?utm_source=github.comutm_medium=referralutm_content=apache/kylinutm_campaign=Badge_Grade)
 
 > Extreme OLAP Engine for Big Data

[03/13] kylin git commit: minor, fix cubeplanner toggle

2018-01-21 Thread lidong

minor, fix cubeplanner toggle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/18673d09
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/18673d09
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/18673d09

Branch: refs/heads/KYLIN-2881-review
Commit: 18673d096d57df217bdaf4c9f93759dbf122d68f
Parents: 91c76dd
Author: lidongsjtu 
Authored: Tue Jan 9 20:03:16 2018 +0800
Committer: lidongsjtu 
Committed: Tue Jan 9 20:03:16 2018 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java| 7 ++-
 webapp/app/js/controllers/page.js | 3 ++-
 webapp/app/partials/header.html   | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/18673d09/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ef9b7ab..341ca3f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1462,12 +1462,17 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.web.export-allow-other", "true"));
 }
 
+public boolean isWebDashboardEnabled() {
+return Boolean.parseBoolean(getOptional("kylin.web.dashboard-enabled", 
"false"));
+}
+
 public String getPropertiesWhiteList() {
 return getOptional("kylin.web.properties.whitelist",
 
"kylin.web.timezone,kylin.query.cache-enabled,kylin.env,kylin.web.hive-limit,kylin.storage.default,kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic,"
 + 
"kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,kylin.web.help.3,"
 + 
"kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,kylin.security.profile,"
-+ 
"kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other");
++ 
"kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other,"
++ 
"kylin.cube.cubeplanner.enabled,kylin.web.dashboard-enabled");
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/18673d09/webapp/app/js/controllers/page.js
--
diff --git a/webapp/app/js/controllers/page.js 
b/webapp/app/js/controllers/page.js
index 4636e65..04a5317 100644
--- a/webapp/app/js/controllers/page.js
+++ b/webapp/app/js/controllers/page.js
@@ -24,7 +24,8 @@ KylinApp.controller('PageCtrl', function ($scope, $q, 
AccessService, $modal, $lo
   kylinConfig.init().$promise.then(function (data) {
 $log.debug(data);
 kylinConfig.initWebConfigInfo();
-$rootScope.isShowCubeplanner = 
kylinConfig.getProperty('kylin.cube.cubeplanner.enabled') === 'true'
+$rootScope.isShowCubeplanner = 
kylinConfig.getProperty('kylin.cube.cubeplanner.enabled') === 'true';
+$rootScope.isShowDashboard = 
kylinConfig.getProperty('kylin.web.dashboard-enabled') === 'true'
   });
   $rootScope.userAction = {
 'islogout': false

http://git-wip-us.apache.org/repos/asf/kylin/blob/18673d09/webapp/app/partials/header.html
--
diff --git a/webapp/app/partials/header.html b/webapp/app/partials/header.html
index 9f1a7e6..32ba591 100644
--- a/webapp/app/partials/header.html
+++ b/webapp/app/partials/header.html
@@ -53,7 +53,7 @@
   
 System
   
-  
+  
 Dashboard

[01/13] kylin git commit: minor, code review [Forced Update!]

2018-01-21 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2881-review ccbd8ac53 -> 61e4d3a76 (forced update)


minor, code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/91c76dd2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/91c76dd2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/91c76dd2

Branch: refs/heads/KYLIN-2881-review
Commit: 91c76dd2d28c6d198bafdb84db04190a7e796159
Parents: 3931157
Author: chenzhx 
Authored: Mon Jan 8 14:01:47 2018 +0800
Committer: chenzhx 
Committed: Mon Jan 8 14:02:11 2018 +0800

--
 webapp/app/js/controllers/cubes.js | 20 +++-
 1 file changed, 19 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/91c76dd2/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js 
b/webapp/app/js/controllers/cubes.js
index e6f4d71..c123f1c 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -597,7 +597,7 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
   if (e.data && e.data.exception) {
 var message = e.data.exception;
 
-if(message.indexOf("Empty cube segment found")!=-1 || 
message.indexOf("Merging segments must not have gaps between")!=-1){
+if(message.indexOf("Empty cube segment found")!=-1){
   var _segment = 
message.substring(message.indexOf(":")+1,message.length-1);
   SweetAlert.swal({
 title:'',
@@ -610,11 +610,29 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
 if (isConfirm) {
   $scope.jobBuildRequest.forceMergeEmptySegment = true;
   $scope.rebuild();
+  delete $scope.jobBuildRequest.forceMergeEmptySegment;
 }
   });
   return;
 }
 
+if(message.indexOf("Merging segments must not have gaps between")!=-1){
+  SweetAlert.swal({
+title:'',
+type:'info',
+text: 'There ares gaps between segments, do you want to merge 
segments forcely ?',
+showCancelButton: true,
+confirmButtonColor: '#DD6B55',
+closeOnConfirm: true
+  }, function (isConfirm) {
+if (isConfirm) {
+  $scope.jobBuildRequest.forceMergeEmptySegment = true;
+  $scope.rebuild();
+  delete $scope.jobBuildRequest.forceMergeEmptySegment;
+}
+  });
+  return;
+}
 var msg = !!(message) ? message : 'Failed to take action.';
 SweetAlert.swal('Oops...', msg, 'error');
   } else {

[06/13] kylin git commit: minor,code review

2018-01-21 Thread lidong

minor,code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/89fffc23
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/89fffc23
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/89fffc23

Branch: refs/heads/KYLIN-2881-review
Commit: 89fffc23b9067f7161df597b26602be0593c67f8
Parents: 1388b73
Author: chenzhx 
Authored: Tue Jan 16 18:08:26 2018 +0800
Committer: chenzhx 
Committed: Tue Jan 16 18:08:26 2018 +0800

--
 webapp/app/partials/cubes/cubes.html | 2 +-
 webapp/app/partials/dashboard/dashboard.html | 6 +++---
 webapp/app/partials/projects/project_create.html | 2 +-
 webapp/app/partials/query/query.html | 4 ++--
 webapp/app/partials/query/query_detail.html  | 4 ++--
 5 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/89fffc23/webapp/app/partials/cubes/cubes.html
--
diff --git a/webapp/app/partials/cubes/cubes.html 
b/webapp/app/partials/cubes/cubes.html
index 8107730..94dc3a6 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -108,7 +108,7 @@
 
 
 
-
+
 Action 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/89fffc23/webapp/app/partials/dashboard/dashboard.html
--
diff --git a/webapp/app/partials/dashboard/dashboard.html 
b/webapp/app/partials/dashboard/dashboard.html
index 6c1c273..0d46098 100644
--- a/webapp/app/partials/dashboard/dashboard.html
+++ b/webapp/app/partials/dashboard/dashboard.html
@@ -130,8 +130,8 @@
   
 
   
-
-  showValue: 
+
+  Show Value: 
 
 
   
@@ -139,7 +139,7 @@
 
   
 
-  
+  
   
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/89fffc23/webapp/app/partials/projects/project_create.html
--
diff --git a/webapp/app/partials/projects/project_create.html 
b/webapp/app/partials/projects/project_create.html
index ceba015..2dc76da 100644
--- a/webapp/app/partials/projects/project_create.html
+++ b/webapp/app/partials/projects/project_create.html
@@ -81,7 +81,7 @@
   
 
 
-  Close
+  Close
   
 Submit
   

http://git-wip-us.apache.org/repos/asf/kylin/blob/89fffc23/webapp/app/partials/query/query.html
--
diff --git a/webapp/app/partials/query/query.html 
b/webapp/app/partials/query/query.html
index e43f7c0..c250cfa 100644
--- a/webapp/app/partials/query/query.html
+++ b/webapp/app/partials/query/query.html
@@ -134,7 +134,7 @@
 
 
+max-size="5" class="pull-right" first-text="First" 
last-text="Last" boundary-links="true" rotate="false">
 
 
 
@@ -178,7 +178,7 @@
 
 
 
+page="cachedFilterQueries.curPage" class="pull-right" 
first-text="First" last-text="Last" boundary-links="true" 
rotate="false">
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/89fffc23/webapp/app/partials/query/query_detail.html
--
diff --git a/webapp/app/partials/query/query_detail.html 
b/webapp/app/partials/query/query_detail.html
index d25477f..63cf5ac 100644
--- a/webapp/app/partials/query/query_detail.html
+++ b/webapp/app/partials/query/query_detail.html
@@ -190,7 +190,7 @@
 Project
 
 
-{{curQuery.project}}
+{{curQuery.project}}
 
 
 
@@ -212,7 +212,7 @@
 
 
 Close
-Save
+Save

kylin git commit: minor, fix cubeplanner toggle

2018-01-09 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master 91c76dd2d -> 18673d096


minor, fix cubeplanner toggle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/18673d09
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/18673d09
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/18673d09

Branch: refs/heads/master
Commit: 18673d096d57df217bdaf4c9f93759dbf122d68f
Parents: 91c76dd
Author: lidongsjtu 
Authored: Tue Jan 9 20:03:16 2018 +0800
Committer: lidongsjtu 
Committed: Tue Jan 9 20:03:16 2018 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java| 7 ++-
 webapp/app/js/controllers/page.js | 3 ++-
 webapp/app/partials/header.html   | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/18673d09/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ef9b7ab..341ca3f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1462,12 +1462,17 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.web.export-allow-other", "true"));
 }
 
+public boolean isWebDashboardEnabled() {
+return Boolean.parseBoolean(getOptional("kylin.web.dashboard-enabled", 
"false"));
+}
+
 public String getPropertiesWhiteList() {
 return getOptional("kylin.web.properties.whitelist",
 
"kylin.web.timezone,kylin.query.cache-enabled,kylin.env,kylin.web.hive-limit,kylin.storage.default,kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic,"
 + 
"kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,kylin.web.help.3,"
 + 
"kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,kylin.security.profile,"
-+ 
"kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other");
++ 
"kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other,"
++ 
"kylin.cube.cubeplanner.enabled,kylin.web.dashboard-enabled");
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/18673d09/webapp/app/js/controllers/page.js
--
diff --git a/webapp/app/js/controllers/page.js 
b/webapp/app/js/controllers/page.js
index 4636e65..04a5317 100644
--- a/webapp/app/js/controllers/page.js
+++ b/webapp/app/js/controllers/page.js
@@ -24,7 +24,8 @@ KylinApp.controller('PageCtrl', function ($scope, $q, 
AccessService, $modal, $lo
   kylinConfig.init().$promise.then(function (data) {
 $log.debug(data);
 kylinConfig.initWebConfigInfo();
-$rootScope.isShowCubeplanner = 
kylinConfig.getProperty('kylin.cube.cubeplanner.enabled') === 'true'
+$rootScope.isShowCubeplanner = 
kylinConfig.getProperty('kylin.cube.cubeplanner.enabled') === 'true';
+$rootScope.isShowDashboard = 
kylinConfig.getProperty('kylin.web.dashboard-enabled') === 'true'
   });
   $rootScope.userAction = {
 'islogout': false

http://git-wip-us.apache.org/repos/asf/kylin/blob/18673d09/webapp/app/partials/header.html
--
diff --git a/webapp/app/partials/header.html b/webapp/app/partials/header.html
index 9f1a7e6..32ba591 100644
--- a/webapp/app/partials/header.html
+++ b/webapp/app/partials/header.html
@@ -53,7 +53,7 @@
   
 System
   
-  
+  
 Dashboard

[1/2] kylin git commit: minor, code review

2018-01-07 Thread chen

Repository: kylin
Updated Branches:
  refs/heads/master 2ee18bc82 -> 91c76dd2d


minor, code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/91c76dd2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/91c76dd2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/91c76dd2

Branch: refs/heads/master
Commit: 91c76dd2d28c6d198bafdb84db04190a7e796159
Parents: 3931157
Author: chenzhx 
Authored: Mon Jan 8 14:01:47 2018 +0800
Committer: chenzhx 
Committed: Mon Jan 8 14:02:11 2018 +0800

--
 webapp/app/js/controllers/cubes.js | 20 +++-
 1 file changed, 19 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/91c76dd2/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js 
b/webapp/app/js/controllers/cubes.js
index e6f4d71..c123f1c 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -597,7 +597,7 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
   if (e.data && e.data.exception) {
 var message = e.data.exception;
 
-if(message.indexOf("Empty cube segment found")!=-1 || 
message.indexOf("Merging segments must not have gaps between")!=-1){
+if(message.indexOf("Empty cube segment found")!=-1){
   var _segment = 
message.substring(message.indexOf(":")+1,message.length-1);
   SweetAlert.swal({
 title:'',
@@ -610,11 +610,29 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
 if (isConfirm) {
   $scope.jobBuildRequest.forceMergeEmptySegment = true;
   $scope.rebuild();
+  delete $scope.jobBuildRequest.forceMergeEmptySegment;
 }
   });
   return;
 }
 
+if(message.indexOf("Merging segments must not have gaps between")!=-1){
+  SweetAlert.swal({
+title:'',
+type:'info',
+text: 'There ares gaps between segments, do you want to merge 
segments forcely ?',
+showCancelButton: true,
+confirmButtonColor: '#DD6B55',
+closeOnConfirm: true
+  }, function (isConfirm) {
+if (isConfirm) {
+  $scope.jobBuildRequest.forceMergeEmptySegment = true;
+  $scope.rebuild();
+  delete $scope.jobBuildRequest.forceMergeEmptySegment;
+}
+  });
+  return;
+}
 var msg = !!(message) ? message : 'Failed to take action.';
 SweetAlert.swal('Oops...', msg, 'error');
   } else {

[1/4] kylin git commit: minor, add maven generated files in .gitignore [Forced Update!]

2018-01-07 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2881-review d5f4feef8 -> ccbd8ac53 (forced update)


minor, add maven generated files in .gitignore


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ee18bc8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ee18bc8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ee18bc8

Branch: refs/heads/KYLIN-2881-review
Commit: 2ee18bc8246d519d3de4910d4ac816a1fda0c8cc
Parents: 18ac702
Author: lidongsjtu 
Authored: Mon Jan 8 13:52:06 2018 +0800
Committer: lidongsjtu 
Committed: Mon Jan 8 13:52:15 2018 +0800

--
 .gitignore | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2ee18bc8/.gitignore
--
diff --git a/.gitignore b/.gitignore
index 3679a13..ca70408 100644
--- a/.gitignore
+++ b/.gitignore
@@ -89,3 +89,4 @@ dist/
 tomcat/
 webapp/app/kylin-servlet.xml
 webapp/app/web.xml
+dependency-reduced-pom.xml

kylin git commit: minor, add maven generated files in .gitignore

2018-01-07 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master 18ac702ad -> 2ee18bc82


minor, add maven generated files in .gitignore


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ee18bc8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ee18bc8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ee18bc8

Branch: refs/heads/master
Commit: 2ee18bc8246d519d3de4910d4ac816a1fda0c8cc
Parents: 18ac702
Author: lidongsjtu 
Authored: Mon Jan 8 13:52:06 2018 +0800
Committer: lidongsjtu 
Committed: Mon Jan 8 13:52:15 2018 +0800

--
 .gitignore | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2ee18bc8/.gitignore
--
diff --git a/.gitignore b/.gitignore
index 3679a13..ca70408 100644
--- a/.gitignore
+++ b/.gitignore
@@ -89,3 +89,4 @@ dist/
 tomcat/
 webapp/app/kylin-servlet.xml
 webapp/app/web.xml
+dependency-reduced-pom.xml

[1/6] kylin git commit: minor,remove GraphService [Forced Update!]

2018-01-07 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2881-review 70c98e5a2 -> d5f4feef8 (forced update)


minor,remove  GraphService


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f86d0678
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f86d0678
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f86d0678

Branch: refs/heads/KYLIN-2881-review
Commit: f86d0678df40d204cc2a0826e5f7e87180a20128
Parents: e44f95e
Author: chenzhx 
Authored: Sat Jan 6 16:37:49 2018 +0800
Committer: chenzhx 
Committed: Sat Jan 6 16:37:49 2018 +0800

--
 webapp/app/js/controllers/query.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f86d0678/webapp/app/js/controllers/query.js
--
diff --git a/webapp/app/js/controllers/query.js 
b/webapp/app/js/controllers/query.js
index ea45d88..dea03f4 100644
--- a/webapp/app/js/controllers/query.js
+++ b/webapp/app/js/controllers/query.js
@@ -432,7 +432,7 @@ KylinApp
 
 
 })
-.controller('QueryResultCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, GraphService, 
kylinConfig) {
+.controller('QueryResultCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, kylinConfig) {
 $scope.isAdminExportAllowed = kylinConfig.isAdminExportAllowed();
 $scope.isNonAdminExportAllowed = kylinConfig.isNonAdminExportAllowed();
 $scope.buildGraphMetadata = function (query) {

[2/6] kylin git commit: minor, add project filter to saved query

2018-01-07 Thread lidong

minor, add project filter to saved query


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ba22a9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ba22a9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ba22a9c

Branch: refs/heads/KYLIN-2881-review
Commit: 2ba22a9ce67b44db62897422e338b29522be67b7
Parents: f86d067
Author: lidongsjtu 
Authored: Sat Jan 6 17:16:09 2018 +0800
Committer: chenzhx 
Committed: Sat Jan 6 17:17:19 2018 +0800

--
 .../kylin/rest/controller/QueryController.java  |  5 +++--
 .../apache/kylin/rest/service/QueryService.java | 22 
 2 files changed, 16 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2ba22a9c/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 4f83780..edb2dd5 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -54,6 +54,7 @@ import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.supercsv.io.CsvListWriter;
 import org.supercsv.io.ICsvListWriter;
@@ -114,9 +115,9 @@ public class QueryController extends BasicController {
 
 @RequestMapping(value = "/saved_queries", method = RequestMethod.GET, 
produces = { "application/json" })
 @ResponseBody
-public List getQueries() throws IOException {
+public List getQueries(@RequestParam(value = "project", required = 
false) String project) throws IOException {
 String creator = 
SecurityContextHolder.getContext().getAuthentication().getName();
-return queryService.getQueries(creator);
+return queryService.getQueries(creator, project);
 }
 
 @RequestMapping(value = "/query/format/{format}", method = 
RequestMethod.GET, produces = { "application/json" })

http://git-wip-us.apache.org/repos/asf/kylin/blob/2ba22a9c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 9789b70..311ad9c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -254,15 +254,20 @@ public class QueryService extends BasicService {
 }
 
 public List getQueries(final String creator) throws IOException {
+return getQueries(creator, null);
+}
+
+public List getQueries(final String creator, final String project) 
throws IOException {
 if (null == creator) {
 return null;
 }
-List queries = new ArrayList();
+List queries = new ArrayList<>();
 QueryRecord record = queryStore.getResource(getQueryKeyById(creator), 
QueryRecord.class,
 QueryRecordSerializer.getInstance());
 if (record != null) {
 for (Query query : record.getQueries()) {
-queries.add(query);
+if (project == null || query.getProject().equals(project))
+queries.add(query);
 }
 }
 return queries;
@@ -414,7 +419,7 @@ public class QueryService extends BasicService {
 
 try (SetThreadName ignored = new SetThreadName("Query %s", 
queryContext.getQueryId())) {
 long startTime = System.currentTimeMillis();
-
+
 SQLResponse sqlResponse = null;
 String sql = sqlRequest.getSql();
 String project = sqlRequest.getProject();
@@ -433,16 +438,16 @@ public class QueryService extends BasicService {
 if (sqlResponse == null && isQueryInspect) {
 sqlResponse = new SQLResponse(null, null, 0, false, 
sqlRequest.getSql());
 }
-
+
 if (sqlResponse == null && isCreateTempStatement) {
 sqlResponse = new SQLResponse(null, null, 0, false, null);
 }
-
+
 if (sqlResponse == null && isQueryCacheEnabled) {

[1/2] kylin git commit: minor, add project filter to saved query

2018-01-06 Thread chen

Repository: kylin
Updated Branches:
  refs/heads/master f86d0678d -> 18ac702ad


minor, add project filter to saved query


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ba22a9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ba22a9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ba22a9c

Branch: refs/heads/master
Commit: 2ba22a9ce67b44db62897422e338b29522be67b7
Parents: f86d067
Author: lidongsjtu 
Authored: Sat Jan 6 17:16:09 2018 +0800
Committer: chenzhx 
Committed: Sat Jan 6 17:17:19 2018 +0800

--
 .../kylin/rest/controller/QueryController.java  |  5 +++--
 .../apache/kylin/rest/service/QueryService.java | 22 
 2 files changed, 16 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2ba22a9c/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 4f83780..edb2dd5 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -54,6 +54,7 @@ import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.supercsv.io.CsvListWriter;
 import org.supercsv.io.ICsvListWriter;
@@ -114,9 +115,9 @@ public class QueryController extends BasicController {
 
 @RequestMapping(value = "/saved_queries", method = RequestMethod.GET, 
produces = { "application/json" })
 @ResponseBody
-public List getQueries() throws IOException {
+public List getQueries(@RequestParam(value = "project", required = 
false) String project) throws IOException {
 String creator = 
SecurityContextHolder.getContext().getAuthentication().getName();
-return queryService.getQueries(creator);
+return queryService.getQueries(creator, project);
 }
 
 @RequestMapping(value = "/query/format/{format}", method = 
RequestMethod.GET, produces = { "application/json" })

http://git-wip-us.apache.org/repos/asf/kylin/blob/2ba22a9c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 9789b70..311ad9c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -254,15 +254,20 @@ public class QueryService extends BasicService {
 }
 
 public List getQueries(final String creator) throws IOException {
+return getQueries(creator, null);
+}
+
+public List getQueries(final String creator, final String project) 
throws IOException {
 if (null == creator) {
 return null;
 }
-List queries = new ArrayList();
+List queries = new ArrayList<>();
 QueryRecord record = queryStore.getResource(getQueryKeyById(creator), 
QueryRecord.class,
 QueryRecordSerializer.getInstance());
 if (record != null) {
 for (Query query : record.getQueries()) {
-queries.add(query);
+if (project == null || query.getProject().equals(project))
+queries.add(query);
 }
 }
 return queries;
@@ -414,7 +419,7 @@ public class QueryService extends BasicService {
 
 try (SetThreadName ignored = new SetThreadName("Query %s", 
queryContext.getQueryId())) {
 long startTime = System.currentTimeMillis();
-
+
 SQLResponse sqlResponse = null;
 String sql = sqlRequest.getSql();
 String project = sqlRequest.getProject();
@@ -433,16 +438,16 @@ public class QueryService extends BasicService {
 if (sqlResponse == null && isQueryInspect) {
 sqlResponse = new SQLResponse(null, null, 0, false, 
sqlRequest.getSql());
 }
-
+
 if (sqlResponse == null && isCreateTempStatement) {
 sqlResponse = new SQLResponse(null, null, 0, false, null);
 }
-

kylin git commit: minor,remove GraphService

2018-01-06 Thread chen

Repository: kylin
Updated Branches:
  refs/heads/master e44f95e63 -> f86d0678d


minor,remove  GraphService


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f86d0678
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f86d0678
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f86d0678

Branch: refs/heads/master
Commit: f86d0678df40d204cc2a0826e5f7e87180a20128
Parents: e44f95e
Author: chenzhx 
Authored: Sat Jan 6 16:37:49 2018 +0800
Committer: chenzhx 
Committed: Sat Jan 6 16:37:49 2018 +0800

--
 webapp/app/js/controllers/query.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f86d0678/webapp/app/js/controllers/query.js
--
diff --git a/webapp/app/js/controllers/query.js 
b/webapp/app/js/controllers/query.js
index ea45d88..dea03f4 100644
--- a/webapp/app/js/controllers/query.js
+++ b/webapp/app/js/controllers/query.js
@@ -432,7 +432,7 @@ KylinApp
 
 
 })
-.controller('QueryResultCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, GraphService, 
kylinConfig) {
+.controller('QueryResultCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, kylinConfig) {
 $scope.isAdminExportAllowed = kylinConfig.isAdminExportAllowed();
 $scope.isNonAdminExportAllowed = kylinConfig.isNonAdminExportAllowed();
 $scope.buildGraphMetadata = function (query) {

[1/2] kylin git commit: minor, format code

2018-01-04 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master fb706854a -> e44f95e63


minor, format code

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7f3fdbe1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7f3fdbe1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7f3fdbe1

Branch: refs/heads/master
Commit: 7f3fdbe11d09a866c2e69776a84eb1c3a8d9d4ff
Parents: fb70685
Author: etherge 
Authored: Thu Jan 4 10:29:50 2018 +0800
Committer: shaofengshi 
Committed: Thu Jan 4 11:25:16 2018 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java  | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7f3fdbe1/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 20ba9ae..e823ebe 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -184,7 +184,6 @@ public class CubeController extends BasicController {
 return cube;
 }
 
-
 /**
  * Get SQL of a Cube
  *
@@ -192,8 +191,7 @@ public class CubeController extends BasicController {
  * @return
  * @throws IOException
  */
-@RequestMapping(value = "/{cubeName}/sql", method = { RequestMethod.GET }, 
produces = {
-"application/json" })
+@RequestMapping(value = "/{cubeName}/sql", method = { RequestMethod.GET }, 
produces = { "application/json" })
 @ResponseBody
 public GeneralResponse getSql(@PathVariable String cubeName) {
 CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -805,7 +803,7 @@ public class CubeController extends BasicController {
 
 if (cuboidList == null || cuboidList.isEmpty()) {
 logger.info("Cannot get recommended cuboid list for cube " + 
cubeName);
-}else {
+} else {
 if (cuboidList.size() < top) {
 logger.info("Require " + top + " recommended cuboids, but only 
" + cuboidList.size() + " is found.");
 }

kylin git commit: minor, add comments in CubeInstance

2017-12-31 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 6f4dbc121 -> edc4397aa


minor, add comments in CubeInstance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/edc4397a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/edc4397a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/edc4397a

Branch: refs/heads/master
Commit: edc4397aa1f168a75a612639376bff89d6424310
Parents: 6f4dbc1
Author: Li Yang 
Authored: Mon Jan 1 07:57:10 2018 +0800
Committer: Li Yang 
Committed: Mon Jan 1 07:57:10 2018 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/edc4397a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 79341cb..a299203 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -578,6 +578,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 }
 
+// For JSON serialization of this attribute, use CubeInstanceResponse
 public long getSizeKB() {
 long sizeKb = 0L;
 
@@ -588,6 +589,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return sizeKb;
 }
 
+// For JSON serialization of this attribute, use CubeInstanceResponse
 public long getInputRecordCount() {
 long sizeRecordCount = 0L;
 
@@ -598,6 +600,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return sizeRecordCount;
 }
 
+// For JSON serialization of this attribute, use CubeInstanceResponse
 public long getInputRecordSizeMB() {
 long sizeRecordSize = 0L;

[40/50] [abbrv] kylin git commit: minor refactor on metadata

2017-12-31 Thread liyang

minor refactor on metadata


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8ddd820
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8ddd820
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8ddd820

Branch: refs/heads/master
Commit: b8ddd820383881fedd6ea617874887e0b0b32734
Parents: 826788a
Author: Hongbin Ma 
Authored: Thu Dec 28 16:15:49 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 28 17:37:18 2017 +0800

--
 .../org/apache/kylin/common/StorageURLTest.java |  18 ++-
 .../common/util/AbstractKylinTestCase.java  |   4 +
 .../apache/kylin/metadata/model/ColumnDesc.java |   5 +-
 .../kylin/metadata/model/DataModelDesc.java |  14 ++-
 .../kylin/metadata/model/DataModelManager.java  |   6 +-
 .../apache/kylin/metadata/model/JoinsTree.java  |  24 +++-
 .../metadata/model/tool/CalciteParser.java  |  37 ++
 .../kylin/metadata/model/JoinsTreeTest.java |   2 +-
 .../kylin/model/tool/CalciteParserTest.java | 122 ---
 .../apache/kylin/query/ITKylinQueryTest.java|   2 +-
 .../org/apache/kylin/query/KylinTestBase.java   |   4 +
 .../test/resources/query/sql_limit/query01.sql  |   2 +-
 .../test/resources/query/sql_limit/query03.sql  |   6 +-
 .../apache/kylin/rest/request/ModelRequest.java |   8 ++
 .../apache/kylin/rest/service/ModelService.java |   5 +-
 .../kylin/rest/util/Log4jConfigListener.java|   1 +
 16 files changed, 121 insertions(+), 139 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b8ddd820/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java 
b/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java
index eaa7548..643cc30 100644
--- a/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java
@@ -63,7 +63,23 @@ public class StorageURLTest {
 }
 
 @Test
+public void testHDFS() {
+{
+StorageURL id = new StorageURL(
+
"master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106");
+
+assertEquals("master_ci_instance", id.getIdentifier());
+assertEquals("hdfs", id.getScheme());
+assertEquals(1, id.getAllParameters().size());
+assertEquals(
+
"master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106",
+id.toString());
+}
+}
+
+@Test
 public void testEdgeCases() {
+
 {
 StorageURL id = new StorageURL("");
 assertEquals("kylin_metadata", id.getIdentifier());
@@ -87,7 +103,7 @@ public class StorageURLTest {
 assertEquals("hello@hbase,a", id.toString());
 }
 }
-
+
 @Test
 public void testValueOfCache() {
 StorageURL id1 = StorageURL.valueOf("hello@hbase");

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8ddd820/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 37790d2..7cbf5b6 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -26,6 +26,10 @@ import org.apache.kylin.common.KylinConfig;
  */
 public abstract class AbstractKylinTestCase {
 
+static {
+System.setProperty("needCheckCC", "true");
+}
+
 public abstract void createTestMetadata(String... overlayMetadataDirs) 
throws Exception;
 
 public abstract void cleanupTestMetadata() throws Exception;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8ddd820/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 03c89c3..b8de534 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -57,6 +57,10 @@ public class ColumnDesc implements Serializable {

[03/50] [abbrv] kylin git commit: minor, refine relization cost calculation

2017-12-31 Thread liyang

minor, refine relization cost calculation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bace2d20
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bace2d20
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bace2d20

Branch: refs/heads/master
Commit: bace2d20b515ef0d458b2579d1923a7fa19d12a8
Parents: 221caaa
Author: Roger Shi 
Authored: Fri Dec 8 21:42:31 2017 +0800
Committer: Roger Shi 
Committed: Mon Dec 11 09:42:38 2017 +0800

--
 .../java/org/apache/kylin/cube/CubeInstance.java | 10 +-
 .../kylin/metadata/realization/IRealization.java |  2 ++
 .../kylin/storage/hybrid/HybridInstance.java |  9 +
 .../kylin/query/routing/RealizationChooser.java  | 19 +--
 .../rest/controller/CubeControllerTest.java  |  3 +--
 5 files changed, 22 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bace2d20/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 17d8a03..d1c5166 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -92,6 +92,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 private String descName;
 @JsonProperty("display_name")
 private String displayName;
+// DEPRECATED: the cost should be calculated in runtime
 // Mark cube priority for query
 @JsonProperty("cost")
 private int cost = 50;
@@ -248,8 +249,15 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 this.displayName = displayName;
 }
 
+@Override
 public int getCost() {
-return cost;
+int countedDimensionNum = getRowKeyColumnCount();
+int c = countedDimensionNum * COST_WEIGHT_DIMENSION + 
getMeasures().size() * COST_WEIGHT_MEASURE;
+for (JoinTableDesc join : getModel().getJoinTables()) {
+if (join.getJoin().isInnerJoin())
+c += CubeInstance.COST_WEIGHT_INNER_JOIN;
+}
+return c;
 }
 
 public void setCost(int cost) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/bace2d20/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index aafc0f0..cc9f1af 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -63,4 +63,6 @@ public interface IRealization extends IStorageAware {
 public boolean supportsLimitPushDown();
 
 public KylinConfig getConfig();
+
+public int getCost();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bace2d20/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index df68e10..14ef524 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -200,6 +200,15 @@ public class HybridInstance extends RootPersistentEntity 
implements IRealization
 }
 
 @Override
+public int getCost() {
+int c = Integer.MAX_VALUE;
+for (IRealization realization : getRealizations()) {
+c = Math.min(realization.getCost(), c);
+}
+return c;
+}
+
+@Override
 public RealizationType getType() {
 return RealizationType.HYBRID;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bace2d20/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java 
b/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index 89859b2..1493c84 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -26,18

[19/50] [abbrv] kylin git commit: minor, refine generateAceResponses.

2017-12-31 Thread liyang

minor, refine  generateAceResponses.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f15da9e6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f15da9e6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f15da9e6

Branch: refs/heads/master
Commit: f15da9e6ca64232ecb6153396b4bb2656c36fbc8
Parents: 3a3d11a
Author: tttMelody <245915...@qq.com>
Authored: Tue Dec 12 20:42:20 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Thu Dec 14 19:07:28 2017 +0800

--
 .../kylin/rest/service/AccessService.java   | 44 +++-
 1 file changed, 43 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f15da9e6/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 3b3b1df..3732c15 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.AclEntity;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
@@ -314,19 +315,60 @@ public class AccessService {
 }
 }
 
-public List generateAceResponses(Acl acl) {
+public List generateAceResponsesByFuzzMatching(Acl 
acl, String nameSeg, boolean isCaseSensitive) {
 if (null == acl) {
 return Collections.emptyList();
 }
 
 List result = new 
ArrayList();
 for (AccessControlEntry ace : acl.getEntries()) {
+if (nameSeg != null && !needAdd(nameSeg, isCaseSensitive, 
getName(ace.getSid( {
+continue;
+}
 result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), 
ace.getPermission(), ace.isGranting()));
 }
 
 return result;
 }
 
+private boolean needAdd(String nameSeg, boolean isCaseSensitive, String 
name) {
+return isCaseSensitive && StringUtils.contains(name, nameSeg)
+|| !isCaseSensitive && StringUtils.containsIgnoreCase(name, 
nameSeg);
+}
+
+private static String getName(Sid sid) {
+if (sid instanceof PrincipalSid) {
+return ((PrincipalSid) sid).getPrincipal();
+} else {
+return ((GrantedAuthoritySid) sid).getGrantedAuthority();
+}
+}
+
+public List generateAceResponses(Acl acl) {
+return generateAceResponsesByFuzzMatching(acl, null, false);
+}
+
+public List getAllAclSids(Acl acl, String type) {
+if (null == acl) {
+return Collections.emptyList();
+}
+
+List result = new ArrayList<>();
+for (AccessControlEntry ace : acl.getEntries()) {
+String name = null;
+if (type.equalsIgnoreCase("user") && ace.getSid() instanceof 
PrincipalSid) {
+name = ((PrincipalSid) ace.getSid()).getPrincipal();
+}
+if (type.equalsIgnoreCase("group") && ace.getSid() instanceof 
GrantedAuthoritySid) {
+name = ((GrantedAuthoritySid) 
ace.getSid()).getGrantedAuthority();
+}
+if (!StringUtils.isBlank(name)) {
+result.add(name);
+}
+}
+return result;
+}
+
 /**
  * Protect admin permission granted to acl owner.
  *

[02/50] [abbrv] kylin git commit: minor, fix bug

2017-12-31 Thread liyang

minor, fix bug


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/221caaab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/221caaab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/221caaab

Branch: refs/heads/master
Commit: 221caaab80a910639c9df3a3d8934de558e76635
Parents: 3ebd15d
Author: Cheng Wang 
Authored: Sat Dec 9 23:43:33 2017 +0800
Committer: Ni Chunen 
Committed: Mon Dec 11 09:15:35 2017 +0800

--
 .../kylin/tool/AbstractInfoExtractor.java   | 28 ++--
 1 file changed, 20 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/221caaab/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
--
diff --git 
a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java 
b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
index 28f3c72..94df3bc 100644
--- a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
@@ -43,17 +43,20 @@ public abstract class AbstractInfoExtractor extends 
AbstractApplication {
 private static final Logger logger = 
LoggerFactory.getLogger(AbstractInfoExtractor.class);
 
 @SuppressWarnings("static-access")
-private static final Option OPTION_DEST = 
OptionBuilder.withArgName("destDir").hasArg().isRequired(true).withDescription("specify
 the dest dir to save the related information").create("destDir");
+private static final Option OPTION_DEST = 
OptionBuilder.withArgName("destDir").hasArg().isRequired(true)
+.withDescription("specify the dest dir to save the related 
information").create("destDir");
 
 @SuppressWarnings("static-access")
-private static final Option OPTION_COMPRESS = 
OptionBuilder.withArgName("compress").hasArg().isRequired(false).withDescription("specify
 whether to compress the output with zip. Default true.").create("compress");
+private static final Option OPTION_COMPRESS = 
OptionBuilder.withArgName("compress").hasArg().isRequired(false)
+.withDescription("specify whether to compress the output with zip. 
Default true.").create("compress");
 
 @SuppressWarnings("static-access")
-private static final Option OPTION_SUBMODULE = 
OptionBuilder.withArgName("submodule").hasArg().isRequired(false).withDescription("specify
 whether this is a submodule of other CLI tool").create("submodule");
+private static final Option OPTION_SUBMODULE = 
OptionBuilder.withArgName("submodule").hasArg().isRequired(false)
+.withDescription("specify whether this is a submodule of other CLI 
tool").create("submodule");
 
 @SuppressWarnings("static-access")
-private static final Option OPTION_PACKAGETYPE = 
OptionBuilder.withArgName("packagetype").hasArg().isRequired(false).withDescription("specify
 the package type").create("packagetype");
-
+private static final Option OPTION_PACKAGETYPE = 
OptionBuilder.withArgName("packagetype").hasArg().isRequired(false)
+.withDescription("specify the package type").create("packagetype");
 
 private static final String DEFAULT_PACKAGE_TYPE = "base";
 private static final String[] COMMIT_SHA1_FILES = { "commit_SHA1", 
"commit.sha1" };
@@ -80,9 +83,17 @@ public abstract class AbstractInfoExtractor extends 
AbstractApplication {
 @Override
 protected void execute(OptionsHelper optionsHelper) throws Exception {
 String exportDest = 
optionsHelper.getOptionValue(options.getOption("destDir"));
-boolean shouldCompress = optionsHelper.hasOption(OPTION_COMPRESS) ? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : true;
-boolean isSubmodule = optionsHelper.hasOption(OPTION_SUBMODULE) ? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_SUBMODULE)) : false;
+boolean shouldCompress = optionsHelper.hasOption(OPTION_COMPRESS)
+? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS))
+: true;
+boolean isSubmodule = optionsHelper.hasOption(OPTION_SUBMODULE)
+? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_SUBMODULE))
+: false;
 packageType = optionsHelper.getOptionValue(OPTION_PACKAGETYPE);
+
+if (packageType == null)
+packageType = DEFAULT_PACKAGE_TYPE;
+
 if (StringUtils.isEmpty(exportDest)) {
 throw new RuntimeException("destDir is not set, exit directly 
without extracting");
 }
@@ -91,7 +102,8 @@ public abstract class AbstractInfoExtractor extends 
AbstractApplication {
 }
 
 // create new folder to contain the output
-

[15/50] [abbrv] kylin git commit: minor, shorten PercentileAggregatorTest, taking too long

2017-12-31 Thread liyang

minor, shorten PercentileAggregatorTest, taking too long


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ff03c8c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ff03c8c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ff03c8c

Branch: refs/heads/master
Commit: 9ff03c8cf647256fa0c3ba3b7adc1947fe73cfe7
Parents: 03e6b8c
Author: Li Yang 
Authored: Sun Dec 10 17:52:29 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 14 16:53:10 2017 +0800

--
 .../apache/kylin/measure/percentile/PercentileAggregatorTest.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff03c8c/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
 
b/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
index 5fcd382..71e2b60 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
@@ -60,7 +60,7 @@ public class PercentileAggregatorTest {
 @Test
 public void testLargeSize() throws Exception {
 for (int compression : compressions) {
-testPercentileSize(2000, null, compression);
+testPercentileSize(200, null, compression);
 }
 }

[39/50] [abbrv] kylin git commit: minor, refine metric.

2017-12-31 Thread liyang

minor, refine metric.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/826788a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/826788a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/826788a3

Branch: refs/heads/master
Commit: 826788a35b717159381e53adb3c5da16eeac360d
Parents: f2722eb
Author: tttMelody <245915...@qq.com>
Authored: Wed Dec 27 20:55:06 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 28 10:34:43 2017 +0800

--
 .../org/apache/kylin/rest/service/QueryService.java | 16 ++--
 1 file changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/826788a3/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 2b76ae3..9afec83 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -409,7 +409,6 @@ public class QueryService extends BasicService {
 HtraceInit.init();
 scope = Trace.startSpan("query life cycle for " + 
queryContext.getQueryId(), Sampler.ALWAYS);
 }
-
 String traceUrl = getTraceUrl(scope);
 
 try (SetThreadName ignored = new SetThreadName("Query %s", 
queryContext.getQueryId())) {
@@ -516,10 +515,11 @@ public class QueryService extends BasicService {
 
 sqlResponse.setTraceUrl(traceUrl);
 logQuery(sqlRequest, sqlResponse);
-
-QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
-QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
-
+try {
+recordMetric(sqlRequest, sqlResponse);
+} catch (Throwable th) {
+logger.warn("Write metric error.", th);
+}
 if (sqlResponse.getIsException())
 throw new 
InternalErrorException(sqlResponse.getExceptionMessage());
 
@@ -528,13 +528,17 @@ public class QueryService extends BasicService {
 } finally {
 BackdoorToggles.cleanToggles();
 QueryContext.reset();
-
 if (scope != null) {
 scope.close();
 }
 }
 }
 
+protected void recordMetric(SQLRequest sqlRequest, SQLResponse 
sqlResponse) throws UnknownHostException {
+QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
+QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
+}
+
 private String getTraceUrl(TraceScope scope) {
 if (scope == null) {
 return null;

[34/50] [abbrv] kylin git commit: minor, restClient support set http timeout

2017-12-31 Thread liyang

minor, restClient support set http timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/63de91df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/63de91df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/63de91df

Branch: refs/heads/master
Commit: 63de91df546c28c4aa745f61619eeb8797eb7b44
Parents: c6e45a7
Author: Chao Long 
Authored: Wed Dec 27 18:12:19 2017 +0800
Committer: Dong Li 
Committed: Wed Dec 27 18:58:35 2017 +0800

--
 .../kylin/common/restclient/RestClient.java | 27 +---
 .../kylin/rest/controller/BasicController.java  |  2 +-
 2 files changed, 24 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/63de91df/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java 
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 67ed9c1..02045ae 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -61,8 +61,8 @@ public class RestClient {
 
 protected static Pattern fullRestPattern = 
Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
 
-private static final int HTTP_CONNECTION_TIMEOUT_MS = 3;
-private static final int HTTP_SOCKET_TIMEOUT_MS = 12;
+private int httpConnectionTimeoutMs = 3;
+private int httpSocketTimeoutMs = 12;
 
 public static final String SCHEME_HTTP = "http://;;
 
@@ -86,6 +86,10 @@ public class RestClient {
  * @param uri "user:pwd@host:port"
  */
 public RestClient(String uri) {
+this(uri, null, null);
+}
+
+public RestClient(String uri, Integer httpConnectionTimeoutMs, Integer 
httpSocketTimeoutMs) {
 Matcher m = fullRestPattern.matcher(uri);
 if (!m.matches())
 throw new IllegalArgumentException("URI: " + uri + " -- does not 
match pattern " + fullRestPattern);
@@ -96,10 +100,25 @@ public class RestClient {
 String portStr = m.group(4);
 int port = Integer.parseInt(portStr == null ? "7070" : portStr);
 
+if (httpConnectionTimeoutMs != null)
+this.httpConnectionTimeoutMs = httpConnectionTimeoutMs;
+if (httpSocketTimeoutMs != null)
+this.httpSocketTimeoutMs = httpSocketTimeoutMs;
+
 init(host, port, user, pwd);
 }
 
 public RestClient(String host, int port, String userName, String password) 
{
+this(host, port, userName, password, null, null);
+}
+
+public RestClient(String host, int port, String userName, String password, 
Integer httpConnectionTimeoutMs,
+Integer httpSocketTimeoutMs) {
+if (httpConnectionTimeoutMs != null)
+this.httpConnectionTimeoutMs = httpConnectionTimeoutMs;
+if (httpSocketTimeoutMs != null)
+this.httpSocketTimeoutMs = httpSocketTimeoutMs;
+
 init(host, port, userName, password);
 }
 
@@ -111,8 +130,8 @@ public class RestClient {
 this.baseUrl = SCHEME_HTTP + host + ":" + port + KYLIN_API_PATH;
 
 final HttpParams httpParams = new BasicHttpParams();
-HttpConnectionParams.setSoTimeout(httpParams, HTTP_SOCKET_TIMEOUT_MS);
-HttpConnectionParams.setConnectionTimeout(httpParams, 
HTTP_CONNECTION_TIMEOUT_MS);
+HttpConnectionParams.setSoTimeout(httpParams, httpSocketTimeoutMs);
+HttpConnectionParams.setConnectionTimeout(httpParams, 
httpConnectionTimeoutMs);
 
 final PoolingClientConnectionManager cm = new 
PoolingClientConnectionManager();
 KylinConfig config = KylinConfig.getInstanceFromEnv();

http://git-wip-us.apache.org/repos/asf/kylin/blob/63de91df/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
index b77309e..539a64c 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
@@ -109,7 +109,7 @@ public class BasicController {
 
 protected void setDownloadResponse(String downloadFile, final 
HttpServletResponse response) {
 File file = new File(downloadFile);
-try (InputStream fileInputStream = new FileInputStream(file); 
OutputStream output = response.getOutputStream();) {
+try

[23/50] [abbrv] kylin git commit: minor, refactor on JoinTree

2017-12-31 Thread liyang

minor, refactor on JoinTree


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9f603d31
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9f603d31
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9f603d31

Branch: refs/heads/master
Commit: 9f603d3188bd20070ea10cbe5fc0eecc28703633
Parents: 26e191a
Author: lidongsjtu 
Authored: Thu Nov 16 23:58:25 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 20 10:09:10 2017 +0800

--
 .../org/apache/kylin/job/JoinedFlatTable.java   |  2 +-
 .../kylin/metadata/model/DataModelDesc.java |  5 +-
 .../apache/kylin/metadata/model/JoinDesc.java   | 28 ---
 .../apache/kylin/metadata/model/JoinsTree.java  | 50 +++-
 4 files changed, 53 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9f603d31/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index d731d4f..d918777 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -178,7 +178,7 @@ public class JoinedFlatTable {
 if (i > 0) {
 sql.append(" AND ");
 }
-sql.append(fk[i].getIdentity() + " = " + 
pk[i].getIdentity());
+sql.append(fk[i].getExpressionInSourceDB() + " = " + 
pk[i].getExpressionInSourceDB());
 }
 sql.append(sep);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9f603d31/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index bf0db73..4dd4399 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -404,11 +404,12 @@ public class DataModelDesc extends RootPersistentEntity {
 alias = alias.toUpperCase();
 join.setAlias(alias);
 
-TableRef ref = new TableRef(this, alias, tableDesc, true);
+boolean isLookup = join.getKind() == TableKind.LOOKUP;
+TableRef ref = new TableRef(this, alias, tableDesc, isLookup);
 
 join.setTableRef(ref);
 addAlias(ref);
-(join.getKind() == TableKind.LOOKUP ? lookupTableRefs : 
factTableRefs).add(ref);
+(isLookup ? lookupTableRefs : factTableRefs).add(ref);
 }
 
 tableNameMap.putAll(aliasMap);

http://git-wip-us.apache.org/repos/asf/kylin/blob/9f603d31/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index b16ad7d..4bf9af3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -187,34 +187,6 @@ public class JoinDesc implements Serializable {
 return true;
 }
 
-// equals() without alias
-public boolean matches(JoinDesc other) {
-if (other == null)
-return false;
-
-if (!this.type.equalsIgnoreCase(other.getType()))
-return false;
-
-// note pk/fk are sorted, sortByFK()
-if (!this.columnDescEquals(foreignKeyColumns, other.foreignKeyColumns))
-return false;
-if (!this.columnDescEquals(primaryKeyColumns, other.primaryKeyColumns))
-return false;
-
-return true;
-}
-
-private boolean columnDescEquals(TblColRef[] a, TblColRef[] b) {
-if (a.length != b.length)
-return false;
-
-for (int i = 0; i < a.length; i++) {
-if (a[i].getColumnDesc().equals(b[i].getColumnDesc()) == false)
-return false;
-}
-return true;
-}
-
 @Override
 public String toString() {
 return "JoinDesc [type=" + type + ", primary_key=" + 
Arrays.toString(primaryKey) + ", foreign_key=" + Arrays.toString(foreignKey) + 
"]";

[13/50] [abbrv] kylin git commit: minor, skip column family ordering check for compatibility

2017-12-31 Thread liyang

minor, skip column family ordering check for compatibility


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/05610dee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/05610dee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/05610dee

Branch: refs/heads/master
Commit: 05610dee7990d819a615efe57e8f9782948ee36f
Parents: 1e35990
Author: Roger Shi 
Authored: Wed Dec 13 12:04:02 2017 +0800
Committer: Roger Shi 
Committed: Wed Dec 13 14:16:02 2017 +0800

--
 .../java/org/apache/kylin/cube/model/CubeDesc.java|  7 +--
 .../test/java/org/apache/kylin/cube/CubeDescTest.java | 14 --
 2 files changed, 5 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/05610dee/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index efb9c59..32578ed 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -1039,8 +1039,11 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 checkState(!measureSet.contains(colMeasureRefs[i]), 
"measure (%s) duplicates", colMeasureRefs[i]);
 measureSet.add(colMeasureRefs[i]);
 
-checkState(measureIndex[i] > lastMeasureIndex, "measure 
(%s) is not in order", colMeasureRefs[i]);
-lastMeasureIndex = measureIndex[i];
+if (storageType > IStorageAware.ID_SHARDED_HBASE) {
+checkState(measureIndex[i] > lastMeasureIndex, 
"measure (%s) is not in order",
+colMeasureRefs[i]);
+lastMeasureIndex = measureIndex[i];
+}
 
 checkEachMeasureExist.set(measureIndex[i]);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/05610dee/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
--
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 36ac5c3..29152fe 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -322,20 +322,6 @@ public class CubeDescTest extends 
LocalFileMetadataTestCase {
 }
 
 @Test
-public void testBadInit13() throws Exception {
-thrown.expect(IllegalStateException.class);
-CubeDesc cubeDesc = 
CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
-MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
-Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
-thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") 
is not in order");
-HBaseColumnDesc colDesc = new HBaseColumnDesc();
-colDesc.setQualifier("M");
-colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", 
"ITEM_COUNT_SUM", "TRANS_CNT" });
-cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = 
colDesc;
-cubeDesc.initMeasureReferenceToColumnFamily();
-}
-
-@Test
 public void testBadInit14() throws Exception {
 thrown.expect(IllegalStateException.class);
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);

[32/50] [abbrv] kylin git commit: minor, add ut for load hive table.

2017-12-31 Thread liyang

minor, add ut for load hive table.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5aa80335
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5aa80335
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5aa80335

Branch: refs/heads/master
Commit: 5aa8033525c9223d305b233928e29ee32193715a
Parents: 19c4eb7
Author: tttMelody <245915...@qq.com>
Authored: Tue Dec 26 11:14:57 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 27 15:05:12 2017 +0800

--
 .../apache/kylin/rest/service/TableService.java | 55 +++-
 .../kylin/rest/service/TableServiceTest.java| 45 
 2 files changed, 75 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5aa80335/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index 6bb446c..d218bc2 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -76,10 +76,6 @@ public class TableService extends BasicService {
 private ModelService modelService;
 
 @Autowired
-@Qualifier("projectService")
-private ProjectService projectService;
-
-@Autowired
 @Qualifier("streamingMgmtService")
 private StreamingService streamingService;
 
@@ -115,27 +111,11 @@ public class TableService extends BasicService {
 
 public String[] loadHiveTablesToProject(String[] tables, String project) 
throws Exception {
 aclEvaluate.checkProjectAdminPermission(project);
-// de-dup
-SetMultimap db2tables = LinkedHashMultimap.create();
-for (String fullTableName : tables) {
-String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
-db2tables.put(parts[0], parts[1]);
-}
+List> allMeta = getAllMeta(tables, 
project);
+return loadHiveTablesToProject(project, allMeta);
+}
 
-// load all tables first
-List> allMeta = Lists.newArrayList();
-ISourceMetadataExplorer explr = 
SourceFactory.getDefaultSource().getSourceMetadataExplorer();
-for (Map.Entry entry : db2tables.entries()) {
-Pair pair = 
explr.loadTableMetadata(entry.getKey(), entry.getValue(), project);
-TableDesc tableDesc = pair.getFirst();
-
Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase()));
-
Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase()));
-
Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase()
 + "." + entry
-.getValue().toUpperCase()));
-TableExtDesc extDesc = pair.getSecond();
-
Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity()));
-allMeta.add(pair);
-}
+String[] loadHiveTablesToProject(String project, List> allMeta) throws Exception {
 
 // do schema check
 TableMetadataManager metaMgr = getTableManager();
@@ -181,7 +161,32 @@ public class TableService extends BasicService {
 addTableToProject(result, project);
 return result;
 }
-
+
+private List> getAllMeta(String[] tables, 
String project) throws Exception {
+// de-dup
+SetMultimap db2tables = LinkedHashMultimap.create();
+for (String fullTableName : tables) {
+String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
+db2tables.put(parts[0], parts[1]);
+}
+
+// load all tables first
+List> allMeta = Lists.newArrayList();
+ISourceMetadataExplorer explr = 
SourceFactory.getDefaultSource().getSourceMetadataExplorer();
+for (Map.Entry entry : db2tables.entries()) {
+Pair pair = 
explr.loadTableMetadata(entry.getKey(), entry.getValue(), project);
+TableDesc tableDesc = pair.getFirst();
+
Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase()));
+
Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase()));
+
Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase()
 + "." + entry
+

[09/50] [abbrv] kylin git commit: minor, print the exception in log

2017-12-31 Thread liyang

minor, print the exception in log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/15dd1c12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/15dd1c12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/15dd1c12

Branch: refs/heads/master
Commit: 15dd1c12bc9b64981e14a6dbe2a28cd987acb588
Parents: f2f487f
Author: Cheng Wang 
Authored: Mon Dec 11 16:47:12 2017 +0800
Committer: Li Yang 
Committed: Mon Dec 11 02:56:17 2017 -0600

--
 .../org/apache/kylin/job/impl/threadpool/DefaultScheduler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/15dd1c12/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 8f4ee37..b87a839 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -126,7 +126,7 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 + nStopped + " stopped, " + nReady + " ready, " + 
nSUCCEED + " already succeed, " + nError
 + " error, " + nDiscarded + " discarded, " + nOthers + 
" others");
 } catch (Exception e) {
-logger.warn("Job Fetcher caught a exception " + e);
+logger.warn("Job Fetcher caught a exception ", e);
 }
 }
 }

[41/50] [abbrv] kylin git commit: minor, add log

2017-12-31 Thread liyang

minor, add log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4eb6cd41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4eb6cd41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4eb6cd41

Branch: refs/heads/master
Commit: 4eb6cd41b31401542091cab7a8aa6f5b38ca8524
Parents: b8ddd82
Author: Hongbin Ma 
Authored: Thu Dec 28 18:51:40 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 28 18:53:55 2017 +0800

--
 .../main/java/org/apache/kylin/metadata/model/DataModelManager.java | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4eb6cd41/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
index a83f2c3..7f9fc5e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
@@ -245,6 +245,7 @@ public class DataModelManager {
 prj.getModels().add(name);
 
 desc.setOwner(owner);
+logger.info("Saving Model {} to Project {} with {} as owner", 
desc.getName(), projectName, owner);
 desc = saveDataModelDesc(desc);
 
 } finally {

[01/50] [abbrv] kylin git commit: Minor, adapt DefaultQueryTransformer's function scope to calcite supported

2017-12-31 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 5f2eff68d -> 4d50b2697


Minor, adapt DefaultQueryTransformer's function scope to calcite supported


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ebd15d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ebd15d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ebd15d5

Branch: refs/heads/master
Commit: 3ebd15d522ee12103badd0547d9316a2e3314137
Parents: d173c7e
Author: Yifan Zhang 
Authored: Fri Dec 8 17:47:02 2017 +0800
Committer: Yifan Zhang 
Committed: Sun Dec 10 22:25:14 2017 +0800

--
 .../org/apache/kylin/query/util/DefaultQueryTransformer.java | 6 --
 .../apache/kylin/query/util/DefaultQueryTransformerTest.java | 8 
 2 files changed, 12 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ebd15d5/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java 
b/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
index b21f410..671d9db 100644
--- 
a/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
+++ 
b/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
@@ -43,7 +43,7 @@ public class DefaultQueryTransformer implements 
IQueryTransformer {
 private static final Pattern PTN_INTERVAL = Pattern.compile(
 "interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)",
 Pattern.CASE_INSENSITIVE);
-private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}",
+private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + SM + "(EXTRACT\\(.*?\\))" + "\\}",
 Pattern.CASE_INSENSITIVE);
 private static final Pattern PIN_SUM_OF_CAST = Pattern.compile(S0 + "SUM" 
+ S0 + "\\(" + S0 + "CAST" + S0 + "\\("
 + S0 + "([^\\s,]+)" + S0 + "AS" + SM + "DOUBLE" + S0 + "\\)" + S0 
+ "\\)", Pattern.CASE_INSENSITIVE);
@@ -76,7 +76,9 @@ public class DefaultQueryTransformer implements 
IQueryTransformer {
 }
 
 // Case {fn EXTRACT(...) }
-// Use non-greedy regrex matching to remove escape functions
+// Use non-greedy regex matching to remove escape functions
+// Notice: Only unsupported escape function need to be handled
+// Reference: 
https://calcite.apache.org/docs/reference.html#jdbc-function-escape
 while (true) {
 m = PTN_HAVING_ESCAPE_FUNCTION.matcher(sql);
 if (!m.find())

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ebd15d5/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
--
diff --git 
a/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
 
b/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
index 5a15443..8ce1017 100644
--- 
a/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
+++ 
b/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
@@ -92,5 +92,13 @@ public class DefaultQueryTransformerTest {
 assertTrue("select sum(LSTG_SITE_ID), sum(price) from KYLIN_SALES 
group by LSTG_SITE_ID"
 .equalsIgnoreCase(correctSql));
 }
+
+@Test
+public void functionEscapeTransform() throws Exception {
+DefaultQueryTransformer transformer = new DefaultQueryTransformer();
 
+String fnConvertSumSql = "select {fn EXTRACT(YEAR from PART_DT)} from 
KYLIN_SALES";
+String correctSql = transformer.transform(fnConvertSumSql, "", "");
+assertTrue("select EXTRACT(YEAR from PART_DT) from 
KYLIN_SALES".equalsIgnoreCase(correctSql));
+}
 }
\ No newline at end of file

[18/50] [abbrv] kylin git commit: minor, change the way that get DefaultScheduler's instance.

2017-12-31 Thread liyang

minor, change the way that get DefaultScheduler's instance.

minor, refine log when lost ZK connection.

minor, remove DefaultScheduler.destroyInstance().


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a3d11a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a3d11a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a3d11a3

Branch: refs/heads/master
Commit: 3a3d11a319024fb7eccf26eb63f7c41891d66080
Parents: 9ff03c8
Author: tttMelody <245915...@qq.com>
Authored: Wed Dec 13 14:44:52 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Thu Dec 14 18:00:18 2017 +0800

--
 .../job/impl/threadpool/DefaultScheduler.java   | 29 +---
 .../job/impl/threadpool/BaseSchedulerTest.java  | 15 +-
 .../kylin/provision/BuildCubeWithEngine.java|  3 +-
 .../kylin/provision/BuildCubeWithStream.java|  3 +-
 4 files changed, 16 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3d11a3/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index ec5f552..327a793 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -62,11 +62,11 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 volatile boolean fetchFailed = false;
 private JobEngineConfig jobEngineConfig;
 
-private static DefaultScheduler INSTANCE = null;
+private static volatile DefaultScheduler INSTANCE = null;
 
 public DefaultScheduler() {
 if (INSTANCE != null) {
-throw new IllegalStateException("DefaultScheduler has been 
initiated.");
+throw new IllegalStateException("DefaultScheduler has been 
initiated. Use getInstance() instead.");
 }
 }
 
@@ -175,14 +175,11 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 }
 }
 
-public static DefaultScheduler getInstance() {
-return INSTANCE;
-}
-
 @Override
 public void stateChanged(CuratorFramework client, ConnectionState 
newState) {
 if ((newState == ConnectionState.SUSPENDED) || (newState == 
ConnectionState.LOST)) {
 try {
+logger.info("ZK Connection state change to " + newState + ", 
shutdown default scheduler.");
 shutdown();
 } catch (SchedulerException e) {
 throw new RuntimeException("failed to shutdown scheduler", e);
@@ -190,23 +187,11 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 }
 }
 
-public synchronized static DefaultScheduler createInstance() {
-destroyInstance();
-INSTANCE = new DefaultScheduler();
-return INSTANCE;
-}
-
-public synchronized static void destroyInstance() {
-DefaultScheduler tmp = INSTANCE;
-INSTANCE = null;
-if (tmp != null) {
-try {
-tmp.shutdown();
-} catch (SchedulerException e) {
-logger.error("error stop DefaultScheduler", e);
-throw new RuntimeException(e);
-}
+public synchronized static DefaultScheduler getInstance() {
+if (INSTANCE == null) {
+INSTANCE = new DefaultScheduler();
 }
+return INSTANCE;
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3d11a3/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index e0a542e..6786d31 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -29,7 +29,7 @@ import org.apache.kylin.job.execution.ExecutableManager;
 import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.lock.MockJobLock;
 import org.junit.After;
-import org.junit.Before;
+import org.junit.BeforeClass;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,16 +39,16 @@ public abstract class BaseSchedulerTest extends 
LocalFileMetadataTestCase {
 
 private static final Logger logger = 
LoggerFactory.getLogger(BaseSchedulerTest.class);
 
-private DefaultScheduler

kylin git commit: minor, display dashboard and cube planner if enabled in KylinConfig

2017-12-25 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master 1d1627a9a -> 5f2eff68d


minor, display dashboard and cube planner if enabled in KylinConfig

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f2eff68
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f2eff68
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f2eff68

Branch: refs/heads/master
Commit: 5f2eff68d80ea6264d7590e14c052114c3cd6b74
Parents: 1d1627a
Author: LauraXia123 
Authored: Tue Dec 26 13:43:21 2017 +0800
Committer: lidongsjtu 
Committed: Tue Dec 26 14:02:35 2017 +0800

--
 webapp/app/js/controllers/cube.js  | 3 ++-
 webapp/app/js/controllers/page.js  | 1 +
 webapp/app/partials/cubes/cube_detail.html | 2 +-
 webapp/app/partials/header.html| 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f2eff68/webapp/app/js/controllers/cube.js
--
diff --git a/webapp/app/js/controllers/cube.js 
b/webapp/app/js/controllers/cube.js
index c0e5efb..35911d3 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -18,12 +18,13 @@
 
 'use strict';
 
-KylinApp.controller('CubeCtrl', function ($scope, AccessService, 
MessageService, CubeService, cubeConfig, TableService, ModelGraphService, 
UserService,SweetAlert,loadingRequest,modelsManager,$modal,cubesManager, 
$location) {
+KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, 
MessageService, CubeService, cubeConfig, TableService, ModelGraphService, 
UserService,SweetAlert,loadingRequest,modelsManager,$modal,cubesManager, 
$location) {
 $scope.newAccess = null;
 $scope.state = {jsonEdit: false};
 
 $scope.modelsManager = modelsManager;
 $scope.cubesManager = cubesManager;
+$scope.isShowCubeplanner = $rootScope.isShowCubeplanner;
 
 $scope.buildGraph = function (cube) {
ModelGraphService.buildTree(cube);

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f2eff68/webapp/app/js/controllers/page.js
--
diff --git a/webapp/app/js/controllers/page.js 
b/webapp/app/js/controllers/page.js
index 8008225..83cc798 100644
--- a/webapp/app/js/controllers/page.js
+++ b/webapp/app/js/controllers/page.js
@@ -24,6 +24,7 @@ KylinApp.controller('PageCtrl', function ($scope, $q, 
AccessService, $modal, $lo
   kylinConfig.init().$promise.then(function (data) {
 $log.debug(data);
 kylinConfig.initWebConfigInfo();
+$rootScope.isShowCubeplanner = 
kylinConfig.getProperty('kylin.cube.cubeplanner.enabled') === 'true'
   });
   $rootScope.userAction = {
 'islogout': false

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f2eff68/webapp/app/partials/cubes/cube_detail.html
--
diff --git a/webapp/app/partials/cubes/cube_detail.html 
b/webapp/app/partials/cubes/cube_detail.html
index cc41b63..38806e1 100755
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@ -41,7 +41,7 @@
 ng-if="userService.hasRole('ROLE_ADMIN')">
 Storage
 
-
+
 Planner
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f2eff68/webapp/app/partials/header.html
--
diff --git a/webapp/app/partials/header.html b/webapp/app/partials/header.html
index fd22f9c..9f1a7e6 100644
--- a/webapp/app/partials/header.html
+++ b/webapp/app/partials/header.html
@@ -53,7 +53,7 @@
   
 System
   
-  
+  
 Dashboard

kylin git commit: minor, fix odbc download link on cn page

2017-11-22 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/document 00b7e8fd4 -> e4e91d8ea


minor, fix odbc download link on cn page


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e4e91d8e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e4e91d8e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e4e91d8e

Branch: refs/heads/document
Commit: e4e91d8ea8c86b10564e893e936b021fecc30b65
Parents: 00b7e8f
Author: lidongsjtu 
Authored: Thu Nov 23 11:19:58 2017 +0800
Committer: lidongsjtu 
Committed: Thu Nov 23 11:19:58 2017 +0800

--
 website/download/index.cn.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e4e91d8e/website/download/index.cn.md
--
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index e2ee352..0f02339 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -25,7 +25,7 @@ title: ä¸è½½
 Apache Kylinçæ§çæ¬å¯ä»¥ä» 
[https://archive.apache.org/dist/kylin/](https://archive.apache.org/dist/kylin/)
 ä¸è½½ã
 
 ## ODBC é©±å¨ç¨åº
-* [Kylin ODBC é©±å¨ 
v2.2.0](http://kylin.apache.org/download/KylinODBCDriver-2.2.0.zip)  
ï¼ä¸ææ Kylin çæ¬å¼å®¹ï¼
+* [Kylin ODBC é©±å¨ 
v2.1.0](http://kylin.apache.org/download/KylinODBCDriver-2.1.0.zip)  
ï¼ä¸ææ Kylin çæ¬å¼å®¹ï¼
 * [Kylin ODBC é©±å¨ 
v1.6](http://kylin.apache.org/download/KylinODBCDriver-1.6.zip)  ï¼ä¸ææ 
Kylin çæ¬å¼å®¹ï¼
 * [Kylin ODBC é©±å¨ 
v1.5](http://kylin.apache.org/download/KylinODBCDriver-1.5.zip)

kylin git commit: minor update

2017-11-13 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document 165de872c -> 303e13383


minor update


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/303e1338
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/303e1338
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/303e1338

Branch: refs/heads/document
Commit: 303e13383e909de0f3cff486d9510074eb94c503
Parents: 165de87
Author: shaofengshi 
Authored: Mon Nov 13 22:56:29 2017 +0800
Committer: shaofengshi 
Committed: Mon Nov 13 22:56:29 2017 +0800

--
 website/_docs21/install/kylin_aws_emr.md | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/303e1338/website/_docs21/install/kylin_aws_emr.md
--
diff --git a/website/_docs21/install/kylin_aws_emr.md 
b/website/_docs21/install/kylin_aws_emr.md
index f594a16..bccb624 100644
--- a/website/_docs21/install/kylin_aws_emr.md
+++ b/website/_docs21/install/kylin_aws_emr.md
@@ -52,6 +52,7 @@ If using HDFS as Kylin working directory, you can leave 
configurations unchanged
 ```
 kylin.env.hdfs-working-dir=/kylin
 ```
+
 This will be very similar as on-premises deployment.
 
 - Use S3 as "kylin.env.hdfs-working-dir"
@@ -65,6 +66,14 @@ kylin.storage.hbase.cluster-fs=s3://yourbucket
 ```
 Then Kylin will use S3 for Cube building, big metadata file and Cube. The 
performance might be slower than HDFS.
 
+- Create the working-dir folder if it doesn't exist
+
+```
+hadoop fs -mkdir /kylin 
+or
+hadoop fs -mkdir s3://yourbucket/kylin
+```
+
 ### Start Kylin
 
 The start is the same as on normal Hadoop:
@@ -75,7 +84,7 @@ $KYLIN_HOME/bin/sample.sh
 $KYLIN_HOME/bin/kylin.sh start
 ```
 
-Don't forget to open the 7070 port access in the security group for EMR 
master, "ElasticMapReduce-master", or with SSH tunnel to the master node, then 
you can access Kylin Web GUI at http://\:7070/kylin
+Don't forget to enable the 7070 port access in the security group for EMR 
master - "ElasticMapReduce-master", or with SSH tunnel to the master node, then 
you can access Kylin Web GUI at http://\:7070/kylin
 
 Build the sample Cube, and then run queries when the Cube is ready. You can 
browse S3 to see whether the data is safely persisted.

[1/2] kylin git commit: minor update on ldap and sso doc

2017-11-13 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document 7dc21fb8e -> 165de872c


minor update on ldap and sso doc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e24d706a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e24d706a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e24d706a

Branch: refs/heads/document
Commit: e24d706aa3c9696bfc504a8412e6a7e0ed63e91a
Parents: 7dc21fb
Author: shaofengshi 
Authored: Mon Nov 13 21:27:39 2017 +0800
Committer: shaofengshi 
Committed: Mon Nov 13 21:27:39 2017 +0800

--
 website/_docs20/howto/howto_ldap_and_sso.md | 2 +-
 website/_docs21/howto/howto_ldap_and_sso.md | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e24d706a/website/_docs20/howto/howto_ldap_and_sso.md
--
diff --git a/website/_docs20/howto/howto_ldap_and_sso.md 
b/website/_docs20/howto/howto_ldap_and_sso.md
index 8085f39..8581d3e 100644
--- a/website/_docs20/howto/howto_ldap_and_sso.md
+++ b/website/_docs20/howto/howto_ldap_and_sso.md
@@ -11,7 +11,7 @@ Kylin supports LDAP authentication for enterprise or 
production deployment; This
 
  Configure LDAP server info
 
-Firstly, provide LDAP URL, and username/password if the LDAP server is 
secured; The password in kylin.properties need be encrypted; You can run the 
following command to get the encrypted value (please note, the password's 
length should be less than 16 characters, see 
[KYLIN-2416](https://issues.apache.org/jira/browse/KYLIN-2416)):
+Firstly, provide LDAP URL, and username/password if the LDAP server is 
secured; The password in kylin.properties need be encrypted; You can run the 
following command to get the encrypted value:
 
 ```
 cd $KYLIN_HOME/tomcat/webapps/kylin/WEB-INF/lib

http://git-wip-us.apache.org/repos/asf/kylin/blob/e24d706a/website/_docs21/howto/howto_ldap_and_sso.md
--
diff --git a/website/_docs21/howto/howto_ldap_and_sso.md 
b/website/_docs21/howto/howto_ldap_and_sso.md
index 98cdba6..945a9fc 100644
--- a/website/_docs21/howto/howto_ldap_and_sso.md
+++ b/website/_docs21/howto/howto_ldap_and_sso.md
@@ -11,11 +11,11 @@ Kylin supports LDAP authentication for enterprise or 
production deployment; This
 
  Configure LDAP server info
 
-Firstly, provide LDAP URL, and username/password if the LDAP server is 
secured; The password in kylin.properties need be encrypted; You can run the 
following command to get the encrypted value (please note, the password's 
length should be less than 16 characters, see 
[KYLIN-2416](https://issues.apache.org/jira/browse/KYLIN-2416)):
+Firstly, provide LDAP URL, and username/password if the LDAP server is 
secured; The password in kylin.properties need be encrypted; You can run the 
following command to get the encrypted value:
 
 ```
 cd $KYLIN_HOME/tomcat/webapps/kylin/WEB-INF/lib
-java -classpath 
kylin-server-base-1.6.0.jar:spring-beans-3.2.17.RELEASE.jar:spring-core-3.2.17.RELEASE.jar:commons-codec-1.7.jar
 org.apache.kylin.rest.security.PasswordPlaceholderConfigurer AES 

+java -classpath 
kylin-server-base-\.jar:spring-beans-3.2.17.RELEASE.jar:spring-core-3.2.17.RELEASE.jar:commons-codec-1.7.jar
 org.apache.kylin.rest.security.PasswordPlaceholderConfigurer AES 

 ```
 
 Config them in the conf/kylin.properties:

kylin git commit: minor fix for log

2017-11-08 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document 76d76a6c1 -> 7dc21fb8e


minor fix for log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7dc21fb8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7dc21fb8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7dc21fb8

Branch: refs/heads/document
Commit: 7dc21fb8e460d998ee24c7d261f6f95eea0e1786
Parents: 76d76a6
Author: shaofengshi 
Authored: Thu Nov 9 09:50:47 2017 +0800
Committer: shaofengshi 
Committed: Thu Nov 9 09:50:47 2017 +0800

--
 website/community/poweredby.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7dc21fb8/website/community/poweredby.md
--
diff --git a/website/community/poweredby.md b/website/community/poweredby.md
index b5ac320..e0e67ee 100644
--- a/website/community/poweredby.md
+++ b/website/community/poweredby.md
@@ -53,5 +53,5 @@ __Companies & Organizations__
 * Apache Kylin is used as a OLAP engine at Wubaâs big data platform, 
powering data analysis for recommend engine, search engine, advertisement 
engine, BI, etc. More than 100 Cubes are daily built under crontab; the query 
statistics shows that more than 95% queries could get response in less than one 
second, which provides better user experience than traditional tools. AndÂ the 
resourceÂ bottleneck could be simply solved by adding machines, which isÂ 
especial friendly to administrator.
 * [4399.com](http://www.4399.com/)
 * Apache Kylin is used at 4399.com for Big Data Analytics on Hadoop, which 
let us query massive data set at sub-second latency.
-* [zteä¸å´](http://www.zte.com.cn/)
+* [ZTE ä¸å´](http://www.zte.com.cn/)
 * Apache Kylin is one of the foundational components of ZTE's Big Data 
Platform. ZTE uses Apache Kylin to provide OLAP capabilities to customers in 
the  telecom, government, banking, finance and other industries.
\ No newline at end of file

kylin git commit: minor, update calcite dependency

2017-11-03 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master 11d8f87af -> 3e21f104d


minor, update calcite dependency


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3e21f104
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3e21f104
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3e21f104

Branch: refs/heads/master
Commit: 3e21f104d2c3d9df896d550e22d907526aa6d5ea
Parents: 11d8f87
Author: lidongsjtu 
Authored: Sat Oct 28 20:53:02 2017 +0800
Committer: lidongsjtu 
Committed: Fri Nov 3 15:35:11 2017 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3e21f104/pom.xml
--
diff --git a/pom.xml b/pom.xml
index ea05c0d..3665f08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,7 +67,7 @@
 
 
 
-1.13.0-kylin-r2-SNAPSHOT
+1.13.0-kylin-r3-SNAPSHOT
 1.10.0

kylin git commit: minor, update KylinVersion

2017-11-03 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/2.2.x 4cf1a98d0 -> 0ae03c839


minor, update KylinVersion


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0ae03c83
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0ae03c83
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0ae03c83

Branch: refs/heads/2.2.x
Commit: 0ae03c8393b87c390b9c7ef49dae5e9c7450026d
Parents: 4cf1a98
Author: lidongsjtu 
Authored: Fri Nov 3 15:34:17 2017 +0800
Committer: lidongsjtu 
Committed: Fri Nov 3 15:34:17 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinVersion.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0ae03c83/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 187aa1f..373b74e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -95,7 +95,7 @@ public class KylinVersion implements Comparable {
 /**
  * Require MANUAL updating kylin version per ANY upgrading.
  */
-private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("2.2.0");
+private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("2.2.1");
 
 private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");

[1/2] kylin git commit: minor, fix how-to-release document

2017-10-29 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/document e21b61cdb -> 52932cf0e


minor, fix how-to-release document


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/42830770
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/42830770
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/42830770

Branch: refs/heads/document
Commit: 428307704e3409aea99007bf43ef31615dad2e1c
Parents: e21b61c
Author: lidongsjtu 
Authored: Sat Oct 28 22:53:48 2017 +0800
Committer: lidongsjtu 
Committed: Sun Oct 29 15:44:42 2017 +0800

--
 website/_dev/howto_release.md | 24 
 1 file changed, 12 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/42830770/website/_dev/howto_release.md
--
diff --git a/website/_dev/howto_release.md b/website/_dev/howto_release.md
index 2807b0e..7fe387f 100644
--- a/website/_dev/howto_release.md
+++ b/website/_dev/howto_release.md
@@ -85,7 +85,7 @@ In the "servers" section, make sure the following servers be 
added, and replace
   #YOUR_APACHE_ID#
   #YOUR_APACHE_PWD#
 
-
+
 
 
   apache.website
@@ -97,7 +97,7 @@ In the "servers" section, make sure the following servers be 
added, and replace
   664
   775
 
-
+
 
 
   stagingSite 
@@ -124,7 +124,7 @@ $ mvn clean
 
 # Make sure all unit tests are passed
 $ mvn test
- 
+
 # Check the `org.apache.kylin.common.KylinVersion` class, ensure the value of 
`CURRENT_KYLIN_VERSION` is the release version. 
 
 # Fix any license issues as reported by target/rat.txt
@@ -133,7 +133,7 @@ $ mvn -Papache-release -DskipTests 
-Dgpg.passphrase=${GPG_PASSPHRASE} install
 
 Optionally, when the dry-run has succeeded, change install to deploy:
 {% highlight bash %}
-$ mvn -Papache-release -Dgpg.passphrase=${GPG_PASSPHRASE} deploy
+$ mvn -Papache-release -DskipTests -Dgpg.passphrase=${GPG_PASSPHRASE} deploy
 {% endhighlight %}
 
 __Prepare and dry run a release__
@@ -157,7 +157,7 @@ $ git clean -xf
 $ mvn clean
 
 # Optionally, do a dry run of the release:prepare step, which sets version 
numbers.
-$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z 
-DdevelopmentVersion=(X.Y.Z+1)-SNAPSHOT -Papache-release 
-Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE}" release:prepare 2>&1 | tee 
/tmp/prepare-dry.log
+$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z 
-DdevelopmentVersion=(X.Y.Z+1)-SNAPSHOT -Papache-release 
-Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE} -DskipTests" release:prepare 
2>&1 | tee /tmp/prepare-dry.log
 {% endhighlight %}
 
 __Check the dry run output:__
@@ -424,13 +424,13 @@ After publish the release, you need generate the binary 
packages and then put th
 * Git checkout the tag for current release; 
 * Make a binary package by refering to [this doc](howto_package.html);
 * Sign the generated binary package with gpg, e.g,:
-{% highlight bash %}
-gpg --armor --output apache-kylin-1.5.0-bin.tar.gz.asc --detach-sig 
apache-kylin-1.5.0-bin.tar.gz
-{% endhighlight %}
+  {% highlight bash %}
+  gpg --armor --output apache-kylin-1.5.0-bin.tar.gz.asc --detach-sig 
apache-kylin-1.5.0-bin.tar.gz
+  {% endhighlight %}
 * Generate the md5 file for the binary package, e.g,:
-{% highlight bash %}
-md5sum < apache-kylin-1.5.0-bin.tar.gz > apache-kylin-1.5.0-bin.tar.gz.md5
-{% endhighlight %}
+  {% highlight bash %}
+  md5sum < apache-kylin-1.5.0-bin.tar.gz > apache-kylin-1.5.0-bin.tar.gz.md5
+  {% endhighlight %}
 * Push the binary package, the signature file and the md5 file to the svn 
__dev__ repo, then run `svn mv  ` to move them 
to svn __release__ repo.
 * For different Hadoop/HBase version, you may need repeat the above steps;
 * Add the files and then commit the svn changes. 
@@ -450,7 +450,7 @@ Send one mail with subject like "[Announce] Apache Kylin 
x.y.z released" to foll
 * Apache Kylin Dev mailing list: d...@kylin.apache.org
 * Apache Kylin User mailing list: u...@kylin.apache.org
 * Apache Announce mailing list: annou...@apache.org
-Please notice to always use your Apache mail address to send this;
+  Please notice to always use your Apache mail address to send this;
 
 Here is a sample of announcement email (by studying Kafka's):

[2/2] kylin git commit: minor, remove beta for spark cubing

2017-10-29 Thread lidong

minor, remove beta for spark cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/52932cf0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/52932cf0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/52932cf0

Branch: refs/heads/document
Commit: 52932cf0efd395c256a2cdecdfeb12b9320eaf88
Parents: 4283077
Author: lidongsjtu 
Authored: Sun Oct 29 15:44:57 2017 +0800
Committer: lidongsjtu 
Committed: Sun Oct 29 15:44:57 2017 +0800

--
 website/_docs21/index.md   | 8 ++--
 website/_docs21/tutorial/cube_spark.md | 8 
 2 files changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/52932cf0/website/_docs21/index.md
--
diff --git a/website/_docs21/index.md b/website/_docs21/index.md
index 12bf1b1..5515cac 100644
--- a/website/_docs21/index.md
+++ b/website/_docs21/index.md
@@ -5,6 +5,7 @@ categories: docs
 permalink: /docs21/index.html
 ---
 
+
 Welcome to Apache Kylinâ¢: Extreme OLAP Engine for Big Data
   
 
@@ -16,6 +17,7 @@ Document of prior versions:
 * [v1.6.x document](/docs16/)
 * [v1.5.x document](/docs15/)
 
+
 Installation & Setup
   
 1. [Hadoop Env](install/hadoop_env.html)
@@ -25,15 +27,16 @@ Installation & Setup
 5. [Run Kylin with Docker](install/kylin_docker.html)
 
 
+
 Tutorial
   
 1. [Quick Start with Sample Cube](tutorial/kylin_sample.html)
 2. [Web Interface](tutorial/web.html)
 3. [Cube Wizard](tutorial/create_cube.html)
-3. [Cube Build and Job Monitoring](tutorial/cube_build_job.html)
+4. [Cube Build and Job Monitoring](tutorial/cube_build_job.html)
 5. [SQL reference: by Apache 
Calcite](http://calcite.apache.org/docs/reference.html)
 6. [Build Cube with Streaming Data](tutorial/cube_streaming.html)
-7. [Build Cube with Spark Engine (beta)](tutorial/cube_spark.html)
+7. [Build Cube with Spark Engine](tutorial/cube_spark.html)
 8. [Cube Build Tuning](tutorial/cube_build_performance.html)
 9. [Enable Query Pushdown](tutorial/query_pushdown.html)
 
@@ -53,6 +56,7 @@ Connectivity and APIs
 10. [Connect from Apache Flink](tutorial/flink.html)
 11. [Connect from Apache Hue](tutorial/hue.html)
 
+
 Operations
   
 1. [Backup/restore Kylin metadata](howto/howto_backup_metadata.html)

http://git-wip-us.apache.org/repos/asf/kylin/blob/52932cf0/website/_docs21/tutorial/cube_spark.md
--
diff --git a/website/_docs21/tutorial/cube_spark.md 
b/website/_docs21/tutorial/cube_spark.md
index 9af44cd..5400309 100644
--- a/website/_docs21/tutorial/cube_spark.md
+++ b/website/_docs21/tutorial/cube_spark.md
@@ -1,6 +1,6 @@
 ---
 layout: docs21
-title:  Build Cube with Spark (beta)
+title:  Build Cube with Spark
 categories: tutorial
 permalink: /docs21/tutorial/cube_spark.html
 ---
@@ -10,7 +10,7 @@ Kylin v2.0 introduces the Spark cube engine, it uses Apache 
Spark to replace Map
 ## Preparation
 To finish this tutorial, you need a Hadoop environment which has Kylin v2.1.0 
or above installed. Here we will use Hortonworks HDP 2.4 Sandbox VM, the Hadoop 
components as well as Hive/HBase has already been started. 
 
-## Install Kylin v2.1.0
+## Install Kylin v2.1.0 or above
 
 Download the Kylin v2.1.0 for HBase 1.x from Kylin's download page, and then 
uncompress the tar ball into */usr/local/* folder:
 
@@ -104,7 +104,7 @@ $KYLIN_HOME/bin/kylin.sh start
 
 {% endhighlight %}
 
-After Kylin is started, access Kylin web, edit the "kylin_sales" cube, in the 
"Advanced Setting" page, change the "Cube Engine" from "MapReduce" to "Spark 
(Beta)":
+After Kylin is started, access Kylin web, edit the "kylin_sales" cube, in the 
"Advanced Setting" page, change the "Cube Engine" from "MapReduce" to "Spark":
 
 
![](/images/tutorial/2.0/Spark-Cubing-Tutorial/1_cube_engine.png)
@@ -166,4 +166,4 @@ Click a specific job, there you will see the detail runtime 
information, that is
 
 If you're a Kylin administrator but new to Spark, suggest you go through 
[Spark documents](https://spark.apache.org/docs/2.1.0/), and don't forget to 
update the configurations accordingly. You can enable Spark [Dynamic Resource 
Allocation](https://spark.apache.org/docs/2.1.0/job-scheduling.html#dynamic-resource-allocation)
 so that it can auto scale/shrink for different work load. Spark's performance 
relies on Cluster's memory and CPU resource, while Kylin's Cube build is a 
heavy task when having a complex data model and a huge dataset to build at one 
time. If your cluster resource couldn't fulfill, errors like "OutOfMemorry" 
will be thrown in Spark executors, so please use it properly. For Cube which 
has UHC dimension, many

kylin git commit: minor, add a simple DataTypeTest

2017-10-28 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 206789289 -> c1f5ed88c


minor, add a simple DataTypeTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1f5ed88
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1f5ed88
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1f5ed88

Branch: refs/heads/master
Commit: c1f5ed88c9f24220a1b75891b7801a7d69dbfd69
Parents: 2067892
Author: Li Yang 
Authored: Sun Oct 29 08:24:39 2017 +0800
Committer: Li Yang 
Committed: Sun Oct 29 08:24:39 2017 +0800

--
 .../metadata/datatype/BigDecimalSerializer.java |  3 +-
 .../metadata/datatype/BooleanSerializer.java|  5 +-
 .../metadata/datatype/DataTypeSerializer.java   |  8 +--
 .../metadata/datatype/DateTimeSerializer.java   |  5 +-
 .../metadata/datatype/DoubleSerializer.java |  1 +
 .../kylin/metadata/datatype/Int4Serializer.java |  5 +-
 .../metadata/datatype/Long8Serializer.java  |  5 +-
 .../kylin/metadata/datatype/LongSerializer.java |  5 +-
 .../metadata/datatype/StringSerializer.java |  1 +
 .../kylin/metadata/datatype/DataTypeTest.java   | 51 
 10 files changed, 74 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c1f5ed88/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
index b5043f5..ba1c4ff 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
@@ -27,9 +27,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * @author yangli9
- * 
  */
+@SuppressWarnings("serial")
 public class BigDecimalSerializer extends DataTypeSerializer {
 
 private static final Logger logger = 
LoggerFactory.getLogger(BigDecimalSerializer.class);

http://git-wip-us.apache.org/repos/asf/kylin/blob/c1f5ed88/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
index 3d485d2..bcbbe49 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
@@ -18,11 +18,12 @@
 
 package org.apache.kylin.metadata.datatype;
 
+import java.nio.ByteBuffer;
+
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.BooleanUtils;
 
-import java.nio.ByteBuffer;
-
+@SuppressWarnings("serial")
 public class BooleanSerializer extends DataTypeSerializer {
 
 public final static String[] TRUE_VALUE_SET = { "true", "t", "on", "yes" };

http://git-wip-us.apache.org/repos/asf/kylin/blob/c1f5ed88/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
index 2de38c0..daa03c1 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
@@ -18,17 +18,19 @@
 
 package org.apache.kylin.metadata.datatype;
 
-import com.google.common.collect.Maps;
-import org.apache.kylin.common.util.BytesSerializer;
-
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.nio.ByteBuffer;
 import java.util.Map;
 
+import org.apache.kylin.common.util.BytesSerializer;
+
+import com.google.common.collect.Maps;
+
 /**
  * Note: the implementations MUST be thread-safe.
  */
+@SuppressWarnings("serial")
 abstract public class DataTypeSerializer implements BytesSerializer, 
java.io.Serializable {
 
 final static Map implementations = Maps.newHashMap();

http://git-wip-us.apache.org/repos/asf/kylin/blob/c1f5ed88/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java

kylin git commit: minor, update calcite dependency

2017-10-28 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/v2.2.0-release 57d97d149 -> c4bc9c5d0


minor, update calcite dependency


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c4bc9c5d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c4bc9c5d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c4bc9c5d

Branch: refs/heads/v2.2.0-release
Commit: c4bc9c5d03ebf12d76bee9826459e806b587d853
Parents: 57d97d1
Author: lidongsjtu 
Authored: Sat Oct 28 22:28:07 2017 +0800
Committer: lidongsjtu 
Committed: Sat Oct 28 22:28:07 2017 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c4bc9c5d/pom.xml
--
diff --git a/pom.xml b/pom.xml
index d8a627c..b637ba1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,7 +67,7 @@
 
 
 
-1.13.0-kylin-r3-SNAPSHOT
+1.13.0-kylin-r3
 1.10.0

kylin git commit: minor, update calcite dependency

2017-10-28 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/v2.2.0-release ed133c203 -> 57d97d149


minor, update calcite dependency


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/57d97d14
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/57d97d14
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/57d97d14

Branch: refs/heads/v2.2.0-release
Commit: 57d97d1498a94e6c8fdb088b3d541f452daa99fe
Parents: ed133c2
Author: lidongsjtu 
Authored: Sat Oct 28 20:53:02 2017 +0800
Committer: lidongsjtu 
Committed: Sat Oct 28 20:53:02 2017 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/57d97d14/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 9693122..d8a627c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,7 +67,7 @@
 
 
 
-1.13.0-kylin-r2-SNAPSHOT
+1.13.0-kylin-r3-SNAPSHOT
 1.10.0

kylin git commit: minor, update calcite dependency

2017-10-28 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/2.2.x ed133c203 -> 57d97d149


minor, update calcite dependency


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/57d97d14
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/57d97d14
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/57d97d14

Branch: refs/heads/2.2.x
Commit: 57d97d1498a94e6c8fdb088b3d541f452daa99fe
Parents: ed133c2
Author: lidongsjtu 
Authored: Sat Oct 28 20:53:02 2017 +0800
Committer: lidongsjtu 
Committed: Sat Oct 28 20:53:02 2017 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/57d97d14/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 9693122..d8a627c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,7 +67,7 @@
 
 
 
-1.13.0-kylin-r2-SNAPSHOT
+1.13.0-kylin-r3-SNAPSHOT
 1.10.0

[08/12] kylin git commit: minor, fix union query for ci

2017-10-24 Thread lidong

minor, fix union query for ci


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a65c3847
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a65c3847
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a65c3847

Branch: refs/heads/2.2.x
Commit: a65c3847e94350251292789d6c449cbed511230b
Parents: 509952f
Author: Roger Shi 
Authored: Sun Oct 22 14:45:27 2017 +0800
Committer: Roger Shi 
Committed: Sun Oct 22 20:57:30 2017 +0800

--
 kylin-it/src/test/resources/query/sql_union/query03.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a65c3847/kylin-it/src/test/resources/query/sql_union/query03.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_union/query03.sql 
b/kylin-it/src/test/resources/query/sql_union/query03.sql
index e92d8e0..44377cb 100644
--- a/kylin-it/src/test/resources/query/sql_union/query03.sql
+++ b/kylin-it/src/test/resources/query/sql_union/query03.sql
@@ -1 +1 @@
-select count(*) as cnt from TEST_KYLIN_FACT where TRANS_ID < 1000 union select 
count(*) as cnt from TEST_KYLIN_FACT where TRANS_ID > 9000
\ No newline at end of file
+select count(*) as cnt from TEST_KYLIN_FACT where ORDER_ID < 100 union select 
count(*) as cnt from TEST_KYLIN_FACT where ORDER_ID > 200
\ No newline at end of file

[09/12] kylin git commit: minor,kylin acl debug (#2876)

2017-10-24 Thread lidong

 minor,kylin acl debug (#2876)

* minor,kylin acl debug
resove conflict

* minor,kylin remove console


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0994a57b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0994a57b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0994a57b

Branch: refs/heads/2.2.x
Commit: 0994a57b922854ad63bb79cbae77d1ea44c3824d
Parents: a65c384
Author: luguosheng1314 <550175...@qq.com>
Authored: Tue Oct 24 02:59:52 2017 -0500
Committer: GitHub 
Committed: Tue Oct 24 02:59:52 2017 -0500

--
 webapp/app/js/controllers/models.js|  2 +-
 webapp/app/js/listeners.js | 10 +++---
 webapp/app/partials/common/access.html | 12 ++--
 webapp/app/partials/cubes/cubes.html   |  6 +++---
 4 files changed, 17 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0994a57b/webapp/app/js/controllers/models.js
--
diff --git a/webapp/app/js/controllers/models.js 
b/webapp/app/js/controllers/models.js
index 1cc0c84..ba7e3f6 100644
--- a/webapp/app/js/controllers/models.js
+++ b/webapp/app/js/controllers/models.js
@@ -132,7 +132,7 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, 
$routeParams, $location,
 var modelstate=false;
 var i=0;
 
-CubeService.list({modelName:model.name}, function (_cubes) {
+
CubeService.list({modelName:model.name,projectName:$scope.projectModel.selectedProject},
 function (_cubes) {
   model.cubes = _cubes;
 
   if (model.cubes.length != 0) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0994a57b/webapp/app/js/listeners.js
--
diff --git a/webapp/app/js/listeners.js b/webapp/app/js/listeners.js
index 959cfca..9d9d473 100644
--- a/webapp/app/js/listeners.js
+++ b/webapp/app/js/listeners.js
@@ -17,14 +17,18 @@
  */
 
 KylinApp.run(function ($rootScope, $http, $location, UserService, 
AuthenticationService, MessageService, $cookieStore, ProjectService, 
ProjectModel, AccessService, SweetAlert, loadingRequest) {
-
   $rootScope.permissions = {
 READ: {name: 'QUERY', value: 'READ', mask: 1},
-MANAGEMENT: {name: 'EDIT', value: 'MANAGEMENT', mask: 32},
+MANAGEMENT: {name: 'MANAGEMENT', value: 'MANAGEMENT', mask: 32},
 OPERATION: {name: 'OPERATION', value: 'OPERATION', mask: 64},
 ADMINISTRATION: {name: 'ADMIN', value: 'ADMINISTRATION', mask: 16}
   };
-
+  $rootScope.permissionsSorted = [
+{name: 'QUERY', value: 'READ', mask: 1},
+{name: 'OPERATION', value: 'OPERATION', mask: 64},
+{name: 'MANAGEMENT', value: 'MANAGEMENT', mask: 32},
+{name: 'ADMIN', value: 'ADMINISTRATION', mask: 16}
+  ];
   $rootScope.$on("$routeChangeStart", function () {
 AuthenticationService.ping(function (data) {
   UserService.setCurUser(data);

http://git-wip-us.apache.org/repos/asf/kylin/blob/0994a57b/webapp/app/partials/common/access.html
--
diff --git a/webapp/app/partials/common/access.html 
b/webapp/app/partials/common/access.html
index ee41e64..9f8d6a5 100644
--- a/webapp/app/partials/common/access.html
+++ b/webapp/app/partials/common/access.html
@@ -50,7 +50,7 @@
 
 Permission 
 
+ng-options="permission.value as permission.name 
for (name, permission) in permissionsSorted">
 
 
 
@@ -87,14 +87,14 @@
 Role
 
 
-QUERY
-EDIT
-OPERATION
-ADMIN
+  QUERY
+  OPERATION
+  MANAGEMENT
+  ADMIN
 
 
 
+ng-options="permission.value as permission.name 
for (name, permission) in permissionsSorted">
 -- select access --
 
 http://git-wip-us.apache.org/repos/asf/kylin/blob/0994a57b/webapp/app/partials/cubes/cubes.html
--
diff --git a/webapp/app/partials/cubes/cubes.html 
b/webapp/app/partials/cubes/cubes.html
index 8528968..38c9342 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -53,7 +53,7 @@
class="fa fa-sort-desc">
 
 Actions
-Admins
+Admins
 
 
 
@@ -106,8 +106,8 @@
 
 
 
-
-

[02/12] kylin git commit: minor, append context info in Rel writer

2017-10-24 Thread lidong

minor, append context info in Rel writer


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c21fe432
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c21fe432
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c21fe432

Branch: refs/heads/2.2.x
Commit: c21fe4325ee6a846a7f2432df8e3330fa5fa556f
Parents: da0d153
Author: Roger Shi 
Authored: Fri Oct 13 10:57:57 2017 +0800
Committer: Roger Shi 
Committed: Thu Oct 19 10:03:36 2017 +0800

--
 .../java/org/apache/kylin/query/relnode/OLAPAggregateRel.java | 7 +++
 .../java/org/apache/kylin/query/relnode/OLAPFilterRel.java| 7 +++
 .../main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java | 7 +++
 .../java/org/apache/kylin/query/relnode/OLAPLimitRel.java | 4 +++-
 .../java/org/apache/kylin/query/relnode/OLAPProjectRel.java   | 7 +++
 .../main/java/org/apache/kylin/query/relnode/OLAPSortRel.java | 6 ++
 .../java/org/apache/kylin/query/relnode/OLAPTableScan.java| 5 -
 .../apache/kylin/query/relnode/OLAPToEnumerableConverter.java | 6 ++
 .../java/org/apache/kylin/query/relnode/OLAPUnionRel.java | 7 +--
 .../java/org/apache/kylin/query/relnode/OLAPWindowRel.java| 4 +++-
 10 files changed, 55 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 1540483..ca7c8ea 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -35,6 +35,7 @@ import org.apache.calcite.plan.RelTrait;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.InvalidRelException;
 import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
 import org.apache.calcite.rel.core.Aggregate;
 import org.apache.calcite.rel.core.AggregateCall;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
@@ -487,4 +488,10 @@ public class OLAPAggregateRel extends Aggregate implements 
OLAPRel {
 this.traitSet = this.traitSet.replace(trait);
 return oldTraitSet;
 }
+
+@Override
+public RelWriter explainTerms(RelWriter pw) {
+return super.explainTerms(pw).item("ctx",
+context == null ? "" : String.valueOf(context.id) + "@" + 
context.realization);
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 8f86ae0..1887ecb 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -36,6 +36,7 @@ import org.apache.calcite.plan.RelOptPlanner;
 import org.apache.calcite.plan.RelTrait;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
 import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
 import org.apache.calcite.rel.type.RelDataType;
@@ -437,4 +438,10 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 this.traitSet = this.traitSet.replace(trait);
 return oldTraitSet;
 }
+
+@Override
+public RelWriter explainTerms(RelWriter pw) {
+return super.explainTerms(pw).item("ctx",
+context == null ? "" : String.valueOf(context.id) + "@" + 
context.realization);
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c21fe432/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 6d0a657..2a97a4c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -43,6 +43,7 @@ import org.apache.calcite.plan.RelTrait;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.InvalidRelException;
 import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
 import

[04/12] kylin git commit: minor, support real admin ldap-group name

2017-10-24 Thread lidong

minor, support real admin ldap-group name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f54daca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f54daca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f54daca

Branch: refs/heads/2.2.x
Commit: 6f54dacaee1dad1df53d28530abdce9ed0e7d49f
Parents: 227668a
Author: lidongsjtu 
Authored: Thu Oct 19 22:32:41 2017 +0800
Committer: lidongsjtu 
Committed: Thu Oct 19 23:12:42 2017 +0800

--
 .../java/org/apache/kylin/rest/security/AuthoritiesPopulator.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6f54daca/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
 
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
index 592791c..dbe2d39 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
@@ -49,7 +49,7 @@ public class AuthoritiesPopulator extends 
DefaultLdapAuthoritiesPopulator {
  */
 public AuthoritiesPopulator(ContextSource contextSource, String 
groupSearchBase, String adminRole, String defaultRole) {
 super(contextSource, groupSearchBase);
-this.adminRoleAsAuthority = new SimpleGrantedAuthority(adminRole);
+this.adminRoleAsAuthority = new 
SimpleGrantedAuthority(adminRole.toUpperCase()); // spring will convert group 
names to uppercase by default
 
 String[] defaultRoles = StringUtils.split(defaultRole, ",");
 if (ArrayUtils.contains(defaultRoles, Constant.ROLE_MODELER)) {

kylin git commit: minor, update doc link on README.md

2017-10-19 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master 37e03a43e -> b53e54f31


minor, update doc link on README.md


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b53e54f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b53e54f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b53e54f3

Branch: refs/heads/master
Commit: b53e54f31f88993e998f7b6656d7874ec5f62ead
Parents: 37e03a4
Author: lidongsjtu 
Authored: Thu Oct 19 23:33:53 2017 +0800
Committer: lidongsjtu 
Committed: Thu Oct 19 23:33:53 2017 +0800

--
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b53e54f3/README.md
--
diff --git a/README.md b/README.md
index 06e5ba4..df8fd3a 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ For more details, see the website 
[http://kylin.apache.org](http://kylin.apache.
 
 Documentation
 =
-Please refer to 
[http://kylin.apache.org/docs16/](http://kylin.apache.org/docs16/).
+Please refer to 
[http://kylin.apache.org/docs21/](http://kylin.apache.org/docs21/).
 
 Get Help

[07/10] kylin git commit: Minor: do not hardcode DefaultQueryTransformer sequence.

2017-10-18 Thread lidong

Minor: do not hardcode DefaultQueryTransformer sequence.

(cherry picked from commit 447c9c27aacbc1524a70da195096e8954c03accf)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/48a08e5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/48a08e5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/48a08e5f

Branch: refs/heads/2.2.x
Commit: 48a08e5f5f112ed2dc0a9408601d265815a42f37
Parents: 5095f1f
Author: Yifan Zhang 
Authored: Tue Oct 10 14:25:20 2017 +0800
Committer: Yifan Zhang 
Committed: Wed Oct 18 13:14:14 2017 +0800

--
 .../main/resources/kylin-defaults.properties|  2 +-
 .../query/util/DefaultQueryTransformer.java | 97 
 .../org/apache/kylin/query/util/QueryUtil.java  | 77 +---
 3 files changed, 99 insertions(+), 77 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/48a08e5f/core-common/src/main/resources/kylin-defaults.properties
--
diff --git a/core-common/src/main/resources/kylin-defaults.properties 
b/core-common/src/main/resources/kylin-defaults.properties
index e3632ae..75436d6 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -200,7 +200,7 @@ 
kylin.query.interceptors=org.apache.kylin.rest.security.TableInterceptor
 kylin.query.escape-default-keyword=false
 
 # Usually should not modify this
-kylin.query.transformers=org.apache.kylin.query.util.KeywordDefaultDirtyHack
+kylin.query.transformers=org.apache.kylin.query.util.DefaultQueryTransformer,org.apache.kylin.query.util.KeywordDefaultDirtyHack
 
 ### SECURITY ###
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/48a08e5f/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java 
b/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
new file mode 100644
index 000..0afe1ed
--- /dev/null
+++ 
b/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.util;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.kylin.query.util.QueryUtil.IQueryTransformer;
+
+public class DefaultQueryTransformer implements IQueryTransformer {
+
+private static final String S0 = "\\s*";
+private static final String S1 = "\\s";
+private static final String SM = "\\s+";
+private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + "GROUP" + 
SM + "BY" + S1,
+Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = 
Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0
++ "COUNT" + S0 + "[(]" + S0 + "1" + S0 + "[)]" + S0 + ">" + S0 + 
"0" + S0 + "[)]?",
+Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + S0 + 
"[(]" + S0 + "[1]" + S0 + "[)]" + S0,
+Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!=" + S0, 
Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_INTERVAL = Pattern.compile(
+"interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)",
+Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}",
+Pattern.CASE_INSENSITIVE);
+
+@Override
+public String transform(String sql, String project, String defaultSchema) {
+Matcher m;
+
+// Case {fn EXTRACT(...) }
+// Use non-greedy regrex matching to remove escape functions
+while (true) {
+m =

[08/10] kylin git commit: Minor: Fix UT

2017-10-18 Thread lidong

Minor: Fix UT

(cherry picked from commit c3390b1ba592eacc06cfde6317594ba4fc8c8462)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ead3c92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ead3c92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ead3c92

Branch: refs/heads/2.2.x
Commit: 3ead3c92e2e933711eb91b8870e74e9ece801ba7
Parents: 48a08e5
Author: Yifan Zhang 
Authored: Tue Oct 10 15:17:39 2017 +0800
Committer: Yifan Zhang 
Committed: Wed Oct 18 13:15:15 2017 +0800

--
 examples/test_case_data/localmeta/kylin.properties | 2 +-
 examples/test_case_data/sandbox/kylin.properties   | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ead3c92/examples/test_case_data/localmeta/kylin.properties
--
diff --git a/examples/test_case_data/localmeta/kylin.properties 
b/examples/test_case_data/localmeta/kylin.properties
index c7dda3f..445e2f9 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -84,7 +84,7 @@ kylin.engine.mr.uhc-reducer-count=3
 
 ### QUERY ###
 
-kylin.query.transformers=org.apache.kylin.query.util.KeywordDefaultDirtyHack
+kylin.query.transformers=org.apache.kylin.query.util.DefaultQueryTransformer,org.apache.kylin.query.util.KeywordDefaultDirtyHack
 
 ### SECURITY ###
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ead3c92/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 619bf99..00fea26 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -197,3 +197,5 @@ 
kylin.engine.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
 #kylin.query.pushdown.jdbc.pool-max-total=8
 #kylin.query.pushdown.jdbc.pool-max-idle=8
 #kylin.query.pushdown.jdbc.pool-min-idle=0
+
+kylin.query.transformers=org.apache.kylin.query.util.DefaultQueryTransformer

[03/10] kylin git commit: minor, add project to cube list api

2017-10-18 Thread lidong

minor, add project to cube list api


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a51ed87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a51ed87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a51ed87

Branch: refs/heads/2.2.x
Commit: 9a51ed87074e996574d2d05b39044371198f4d91
Parents: d20e1d0
Author: lidongsjtu 
Authored: Tue Oct 17 15:21:55 2017 +0800
Committer: lidongsjtu 
Committed: Tue Oct 17 15:38:05 2017 +0800

--
 .../kylin/rest/controller/CubeController.java   | 28 +---
 .../apache/kylin/rest/service/CubeService.java  | 27 ++-
 .../rest/controller/AccessControllerTest.java   | 23 +---
 .../rest/controller/CubeControllerTest.java | 10 +++
 4 files changed, 62 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a51ed87/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b6ec0e2..c3b6e45 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -52,6 +52,7 @@ import org.apache.kylin.rest.exception.NotFoundException;
 import org.apache.kylin.rest.request.CubeRequest;
 import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.request.JobBuildRequest2;
+import org.apache.kylin.rest.response.CubeInstanceResponse;
 import org.apache.kylin.rest.response.EnvelopeResponse;
 import org.apache.kylin.rest.response.GeneralResponse;
 import org.apache.kylin.rest.response.HBaseResponse;
@@ -109,26 +110,34 @@ public class CubeController extends BasicController {
 
 @RequestMapping(value = "", method = { RequestMethod.GET }, produces = { 
"application/json" })
 @ResponseBody
-public List getCubes(@RequestParam(value = "cubeName", 
required = false) String cubeName,
+public List getCubes(@RequestParam(value = 
"cubeName", required = false) String cubeName,
 @RequestParam(value = "modelName", required = false) String 
modelName,
 @RequestParam(value = "projectName", required = false) String 
projectName,
 @RequestParam(value = "limit", required = false) Integer limit,
 @RequestParam(value = "offset", required = false) Integer offset) {
-List cubes;
-cubes = cubeService.listAllCubes(cubeName, projectName, modelName, 
true);
+List cubes = cubeService.listAllCubes(cubeName, 
projectName, modelName, true);
 
-int climit = (null == limit) ? cubes.size() : limit;
+List response = 
Lists.newArrayListWithExpectedSize(cubes.size());
+for (CubeInstance cube : cubes) {
+try {
+response.add(cubeService.createCubeInstanceResponse(cube));
+} catch (Exception e) {
+logger.error("Error creating cube instance response, 
skipping.", e);
+}
+}
+
+int climit = (null == limit) ? response.size() : limit;
 int coffset = (null == offset) ? 0 : offset;
 
-if (cubes.size() <= coffset) {
+if (response.size() <= coffset) {
 return Collections.emptyList();
 }
 
-if ((cubes.size() - coffset) < climit) {
-return cubes.subList(coffset, cubes.size());
+if ((response.size() - coffset) < climit) {
+return response.subList(coffset, response.size());
 }
 
-return cubes.subList(coffset, coffset + climit);
+return response.subList(coffset, coffset + climit);
 }
 
 @RequestMapping(value = "validEncodings", method = { RequestMethod.GET }, 
produces = { "application/json" })
@@ -280,7 +289,7 @@ public class CubeController extends BasicController {
 @RequestMapping(value = "/{cubeName}/rebuild", method = { 
RequestMethod.PUT }, produces = { "application/json" })
 @ResponseBody
 public JobInstance rebuild(@PathVariable String cubeName, @RequestBody 
JobBuildRequest req) {
-return buildInternal(cubeName, new TSRange(req.getStartTime(), 
req.getEndTime()), null, null, null, 
+return buildInternal(cubeName, new TSRange(req.getStartTime(), 
req.getEndTime()), null, null, null,
 req.getBuildType(), req.isForce() || 
req.isForceMergeEmptySegment());
 }
 
@@ -746,5 +755,4 @@ public class CubeController extends BasicController {
 public void setJobService(JobService jobService) {
 this.jobService = jobService;
 }
-
 }

[04/10] kylin git commit: minor, fix UT

2017-10-18 Thread lidong

minor, fix UT



Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32b24008
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32b24008
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32b24008

Branch: refs/heads/2.2.x
Commit: 32b24008fd676bed3301ee1a19d3df271eb6bba5
Parents: 9a51ed8
Author: Dong Li 
Authored: Tue Oct 17 03:58:05 2017 -0500
Committer: lidongsjtu 
Committed: Tue Oct 17 17:00:16 2017 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeControllerTest.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/32b24008/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java 
b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index 950f0b4..8ee4342 100644
--- 
a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ 
b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -124,7 +124,7 @@ public class CubeControllerTest extends ServiceTestBase {
 List cubeInstances = 
cubeController.getCubes(newCubeName, cube.getModelName(), "default",
 1, 0);
 
-CubeInstance cubeInstance = cubeInstances.get(0);
+CubeInstance cubeInstance = 
cubeController.getCube(cubeInstances.get(0).getName());
 
Assert.assertTrue(cubeInstance.getDescriptor().getNotifyList().contains("j...@example.com"));
 Assert.assertTrue(cubeInstance.getCost() == 80);
 cubeController.deleteCube(newCubeName);

[2/4] kylin git commit: minor, fix NPE in ACL ValidateUtil getALlUsers

2017-10-12 Thread lidong

minor, fix NPE in ACL ValidateUtil getALlUsers


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5fbd9525
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5fbd9525
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5fbd9525

Branch: refs/heads/2.2.x
Commit: 5fbd9525d1dbe0892675f94c38df93b625fefa9e
Parents: bd7cb9c
Author: Billy Liu 
Authored: Sat Sep 30 15:04:19 2017 +0800
Committer: Billy Liu 
Committed: Sat Sep 30 16:14:10 2017 +0800

--
 .../src/main/java/org/apache/kylin/rest/util/ValidateUtil.java | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5fbd9525/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
index abb326f..7e513ae 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
@@ -122,8 +122,10 @@ public class ValidateUtil {
 ProjectInstance prj = 
projectService.getProjectManager().getProject(project);
 AclEntity ae = accessService.getAclEntity("ProjectInstance", 
prj.getUuid());
 Acl acl = accessService.getAcl(ae);
-for (AccessControlEntry ace : acl.getEntries()) {
-allUsers.add(((PrincipalSid) ace.getSid()).getPrincipal());
+if (acl != null && acl.getEntries() != null) {
+for (AccessControlEntry ace : acl.getEntries()) {
+allUsers.add(((PrincipalSid) ace.getSid()).getPrincipal());
+}
 }
 return allUsers;
 }

[02/11] kylin git commit: minor, fix NPE in ACL ValidateUtil getALlUsers

2017-10-08 Thread liyang

minor, fix NPE in ACL ValidateUtil getALlUsers


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2415fd9e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2415fd9e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2415fd9e

Branch: refs/heads/master
Commit: 2415fd9e65b7bf483c375eda7d074c6804edb60e
Parents: bd7cb9c
Author: Billy Liu 
Authored: Sat Sep 30 15:04:19 2017 +0800
Committer: Li Yang 
Committed: Sat Sep 30 16:11:29 2017 +0800

--
 .../src/main/java/org/apache/kylin/rest/util/ValidateUtil.java | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2415fd9e/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
index abb326f..7e513ae 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
@@ -122,8 +122,10 @@ public class ValidateUtil {
 ProjectInstance prj = 
projectService.getProjectManager().getProject(project);
 AclEntity ae = accessService.getAclEntity("ProjectInstance", 
prj.getUuid());
 Acl acl = accessService.getAcl(ae);
-for (AccessControlEntry ace : acl.getEntries()) {
-allUsers.add(((PrincipalSid) ace.getSid()).getPrincipal());
+if (acl != null && acl.getEntries() != null) {
+for (AccessControlEntry ace : acl.getEntries()) {
+allUsers.add(((PrincipalSid) ace.getSid()).getPrincipal());
+}
 }
 return allUsers;
 }

[10/15] kylin git commit: minor, add RoutingIndicatorException to expected exception when query pushdown.

2017-09-29 Thread liyang

minor, add RoutingIndicatorException to expected exception when query pushdown.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6dbb8402
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6dbb8402
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6dbb8402

Branch: refs/heads/master
Commit: 6dbb8402e7379c7e59fdf7ce39630a1739b4169d
Parents: 617de5c
Author: tttMelody <245915...@qq.com>
Authored: Wed Sep 27 15:41:09 2017 +0800
Committer: YIMINGXU <100650...@qq.com>
Committed: Wed Sep 27 03:10:02 2017 -0500

--
 .../org/apache/kylin/query/util/PushDownUtil.java| 15 ++-
 .../org/apache/kylin/rest/service/QueryService.java  |  2 +-
 2 files changed, 11 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6dbb8402/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java 
b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index a9f6c5c..2c93bdf 100644
--- a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -133,14 +133,19 @@ public class PushDownUtil {
 Preconditions.checkArgument(sqlException != null);
 Throwable rootCause = ExceptionUtils.getRootCause(sqlException);
 
+//SqlValidatorException is not an excepted exception in the origin 
design.But in the multi pass scene,
+//query pushdown may create tables, and the tables are not in the 
model, so will throw SqlValidatorException.
 boolean isPushDownUpdateEnabled = 
KylinConfig.getInstanceFromEnv().isPushDownUpdateEnabled();
+
 if (!isPushDownUpdateEnabled) {
-return rootCause != null && rootCause instanceof 
NoRealizationFoundException;
+return rootCause != null //
+&& (rootCause instanceof NoRealizationFoundException //
+|| rootCause instanceof 
RoutingIndicatorException); //
 } else {
-return (rootCause != null
-&& (rootCause instanceof NoRealizationFoundException
-|| rootCause instanceof SqlValidatorException
-|| rootCause instanceof RoutingIndicatorException));
+return (rootCause != null //
+&& (rootCause instanceof NoRealizationFoundException //
+|| rootCause instanceof SqlValidatorException //
+|| rootCause instanceof 
RoutingIndicatorException)); //
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6dbb8402/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 33dd4e8..b31fb3b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -413,7 +413,7 @@ public class QueryService extends BasicService {
 if (null == sqlResponse) {
 if (isSelect) {
 sqlResponse = query(sqlRequest);
-} else if (kylinConfig.isPushDownEnabled()) {
+} else if (kylinConfig.isPushDownEnabled() && 
kylinConfig.isPushDownUpdateEnabled()) {
 sqlResponse = update(sqlRequest);
 } else {
 logger.debug(

[02/15] kylin git commit: minor, add switches to enable/disable escape "DEFAULT".

2017-09-29 Thread liyang

minor, add switches to enable/disable escape "DEFAULT".


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/036d70fd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/036d70fd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/036d70fd

Branch: refs/heads/master
Commit: 036d70fd0996ded0119c7c58369701eedac93a13
Parents: 2e845c2
Author: tttMelody <245915...@qq.com>
Authored: Sun Sep 24 18:16:29 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Sun Sep 24 18:32:15 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  10 +-
 .../main/resources/kylin-defaults.properties|  14 +-
 .../org/apache/kylin/metadata/acl/TableACL.java |   2 +-
 .../relnode/OLAPToEnumerableConverter.java  |   8 +-
 .../kylin/query/security/QueryIntercept.java|  64 ---
 .../query/security/QueryInterceptUtil.java  | 170 ---
 .../kylin/query/security/QueryInterceptor.java  |  64 +++
 .../query/security/QueryInterceptorUtil.java| 170 +++
 .../query/util/KeywordDefaultDirtyHack.java |   5 +
 .../apache/kylin/query/util/QueryUtilTest.java  |   2 +
 .../kylin/rest/security/TableIntercept.java |  57 ---
 .../kylin/rest/security/TableInterceptor.java   |  57 +++
 12 files changed, 320 insertions(+), 303 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/036d70fd/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b672a7a..ee05d69 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1090,8 +1090,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptionalStringArray("kylin.query.transformers", new 
String[0]);
 }
 
-public String[] getQueryIntercept() {
-return getOptionalStringArray("kylin.query.intercepts", new String[0]);
+public String[] getQueryInterceptors() {
+return getOptionalStringArray("kylin.query.interceptors", new 
String[0]);
 }
 
 public long getQueryDurationCacheThreshold() {
@@ -1190,7 +1190,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public boolean isTableACLEnabled() {
-return 
Boolean.valueOf(this.getOptional("kylin.query.acl.table-acl-enabled", "true"));
+return 
Boolean.valueOf(this.getOptional("kylin.query.security.table-acl-enabled", 
"true"));
+}
+
+public boolean isEscapeDefaultKeywordEnabled() {
+return 
Boolean.valueOf(this.getOptional("kylin.query.escape-default-keyword", 
"false"));
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/036d70fd/core-common/src/main/resources/kylin-defaults.properties
--
diff --git a/core-common/src/main/resources/kylin-defaults.properties 
b/core-common/src/main/resources/kylin-defaults.properties
index 3bc534b..a317f8e 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -191,6 +191,16 @@ kylin.query.max-scan-bytes=0
 
 kylin.query.cache-enabled=true
 
+# TABLE ACL
+kylin.query.security.table-acl-enabled=true
+
+# Usually should not modify this
+kylin.query.interceptors=org.apache.kylin.rest.security.TableInterceptor
+
+kylin.query.escape-default-keyword=false
+
+# Usually should not modify this
+kylin.query.transformers=org.apache.kylin.query.util.KeywordDefaultDirtyHack
 
 ### SECURITY ###
 
@@ -277,7 +287,3 @@ 
kylin.engine.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
 #kylin.query.pushdown.jdbc.pool-max-total=8
 #kylin.query.pushdown.jdbc.pool-max-idle=8
 #kylin.query.pushdown.jdbc.pool-min-idle=0
-
-### TABLE ACL
-kylin.query.acl.table-acl-enabled=true
-kylin.query.intercepts=org.apache.kylin.rest.security.TableIntercept
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/036d70fd/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
index 0b3ba4d..f4cfd1f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
@@

[07/15] kylin git commit: minor, lazy init ACLUtil

2017-09-29 Thread liyang

minor, lazy init ACLUtil


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/22631a62
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/22631a62
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/22631a62

Branch: refs/heads/master
Commit: 22631a62232c9696ce2532af60c623694468fef0
Parents: b425bb9
Author: tttMelody <245915...@qq.com>
Authored: Tue Sep 26 16:06:13 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Tue Sep 26 20:48:10 2017 +0800

--
 .../main/java/org/apache/kylin/rest/service/UserService.java  | 7 ++-
 .../src/main/java/org/apache/kylin/rest/util/AclUtil.java | 2 ++
 2 files changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/22631a62/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index aa48e1f..b257427 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,6 +30,7 @@ import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.Serializer;
 import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.msg.Message;
 import org.apache.kylin.rest.msg.MsgPicker;
@@ -70,10 +71,6 @@ public class UserService implements UserDetailsManager {
 @Qualifier("accessService")
 private AccessService accessService;
 
-@Autowired
-@Qualifier("projectService")
-private ProjectService projectService;
-
 public boolean isEvictCacheFlag() {
 return evictCacheFlag;
 }
@@ -114,7 +111,7 @@ public class UserService implements UserDetailsManager {
 
 try {
 //revoke user's project permission
-List projectInstances = 
projectService.listProjects(null, null);
+List projectInstances = 
ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).listAllProjects();
 for (ProjectInstance pi : projectInstances) {
 AclEntity ae = accessService.getAclEntity("ProjectInstance", 
pi.getUuid());
 Acl acl = accessService.getAcl(ae);

http://git-wip-us.apache.org/repos/asf/kylin/blob/22631a62/server-base/src/main/java/org/apache/kylin/rest/util/AclUtil.java
--
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/AclUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/AclUtil.java
index 602079e..9ebc8ce 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/AclUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/AclUtil.java
@@ -20,10 +20,12 @@ package org.apache.kylin.rest.util;
 
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.rest.constant.Constant;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Component;
 
+@Lazy
 @Component("aclUtil")
 public class AclUtil {
 String getCurrentUserName() {

[01/15] kylin git commit: minor, wipe query cache when update table ACL.

2017-09-29 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master f11f184b0 -> ef7d29083


minor, wipe query cache when update table ACL.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e845c21
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e845c21
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e845c21

Branch: refs/heads/master
Commit: 2e845c21343e4143bfa5a3d6bb54842808c57275
Parents: 0a8bdc2
Author: tttMelody <245915...@qq.com>
Authored: Fri Sep 22 18:57:41 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Sun Sep 24 17:47:04 2017 +0800

--
 .../kylin/metadata/acl/TableACLManager.java   |  1 +
 .../kylin/metadata/cachesync/Broadcaster.java | 15 +++
 .../org/apache/kylin/rest/request/SQLRequest.java | 18 --
 .../apache/kylin/rest/service/CacheService.java   |  5 +
 .../apache/kylin/rest/service/QueryService.java   |  9 +
 5 files changed, 46 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e845c21/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
index 4804cef..4812e1c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
@@ -100,6 +100,7 @@ public class TableACLManager {
 @Override
 public void onEntityChange(Broadcaster broadcaster, String entity, 
Broadcaster.Event event, String cacheKey) throws IOException {
 reloadTableACL(cacheKey);
+broadcaster.notifyProjectACLUpdate(cacheKey);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e845c21/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 532ae74..ef19379 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -65,6 +65,7 @@ public class Broadcaster {
 public static final String SYNC_ALL = "all"; // the special entity to 
indicate clear all
 public static final String SYNC_PRJ_SCHEMA = "project_schema"; // the 
special entity to indicate project schema has change, e.g. 
table/model/cube_desc update
 public static final String SYNC_PRJ_DATA = "project_data"; // the special 
entity to indicate project data has change, e.g. cube/raw_table update
+public static final String SYNC_PRJ_ACL = "project_acl"; // the special 
entity to indicate query ACL has change, e.g. table_acl/learn_kylin update
 
 // static cached instances
 private static final ConcurrentMap CACHE = new 
ConcurrentHashMap();
@@ -199,6 +200,7 @@ public class Broadcaster {
 addListener(lmap, SYNC_ALL, listener);
 addListener(lmap, SYNC_PRJ_SCHEMA, listener);
 addListener(lmap, SYNC_PRJ_DATA, listener);
+addListener(lmap, SYNC_PRJ_ACL, listener);
 }
 }
 
@@ -223,6 +225,10 @@ public class Broadcaster {
 notifyListener(SYNC_PRJ_DATA, Event.UPDATE, project);
 }
 
+public void notifyProjectACLUpdate(String project) throws IOException {
+notifyListener(SYNC_PRJ_ACL, Event.UPDATE, project);
+}
+
 public void notifyListener(String entity, Event event, String cacheKey) 
throws IOException {
 notifyListener(entity, event, cacheKey, true);
 }
@@ -268,6 +274,12 @@ public class Broadcaster {
 l.onProjectDataChange(this, cacheKey);
 }
 break;
+case SYNC_PRJ_ACL:
+ProjectManager.getInstance(config).clearL2Cache();
+for (Listener l : list) {
+l.onProjectQueryACLChange(this, cacheKey);
+}
+break;
 default:
 for (Listener l : list) {
 l.onEntityChange(this, entity, event, cacheKey);
@@ -332,6 +344,9 @@ public class Broadcaster {
 public void onProjectDataChange(Broadcaster broadcaster, String 
project) throws IOException {
 }
 
+public void onProjectQueryACLChange(Broadcaster broadcaster, String 
project) throws IOException {
+}
+
 public void onEntityChange(Broadcaster

[11/15] kylin git commit: minor, add license header on some query test cases

2017-09-29 Thread liyang

minor, add license header on some query test cases


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3b74cea8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3b74cea8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3b74cea8

Branch: refs/heads/master
Commit: 3b74cea8e610c021240983d68e2ba3e1326e64f0
Parents: 6dbb840
Author: Li Yang 
Authored: Wed Sep 27 15:09:31 2017 +0800
Committer: æ 
Committed: Thu Sep 28 11:08:38 2017 +0800

--
 .../test/resources/query/sql_subquery/query17.sql | 18 ++
 .../test/resources/query/sql_subquery/query18.sql | 18 ++
 .../test/resources/query/sql_subquery/query19.sql | 18 ++
 .../test/resources/query/sql_subquery/query21.sql | 18 ++
 .../test/resources/query/sql_subquery/query22.sql | 18 ++
 .../test/resources/query/sql_subquery/query23.sql | 18 ++
 .../test/resources/query/sql_subquery/query24.sql | 18 ++
 .../test/resources/query/sql_subquery/query25.sql | 18 ++
 .../test/resources/query/sql_subquery/query26.sql | 18 ++
 .../test/resources/query/sql_subquery/query27.sql | 18 ++
 .../test/resources/query/sql_subquery/query28.sql | 18 ++
 .../resources/query/sql_subquery/query29.sql.todo | 18 ++
 .../test/resources/query/sql_subquery/query30.sql | 18 ++
 .../test/resources/query/sql_subquery/query31.sql | 18 ++
 .../test/resources/query/sql_subquery/query32.sql | 18 ++
 .../test/resources/query/sql_subquery/query33.sql | 18 ++
 .../test/resources/query/sql_subquery/query34.sql | 18 ++
 17 files changed, 306 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3b74cea8/kylin-it/src/test/resources/query/sql_subquery/query17.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query17.sql 
b/kylin-it/src/test/resources/query/sql_subquery/query17.sql
index c5833b0..001bfa0 100644
--- a/kylin-it/src/test/resources/query/sql_subquery/query17.sql
+++ b/kylin-it/src/test/resources/query/sql_subquery/query17.sql
@@ -1,3 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
 
 select test_kylin_fact.lstg_format_name,sum(test_kylin_fact.price) as GMV 
  , count(*) as TRANS_CNT

http://git-wip-us.apache.org/repos/asf/kylin/blob/3b74cea8/kylin-it/src/test/resources/query/sql_subquery/query18.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query18.sql 
b/kylin-it/src/test/resources/query/sql_subquery/query18.sql
index 57bb895..99d5aa9 100644
--- a/kylin-it/src/test/resources/query/sql_subquery/query18.sql
+++ b/kylin-it/src/test/resources/query/sql_subquery/query18.sql
@@ -1,3 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
 
 
 select test_kylin_fact.lstg_format_name, sum(test_kylin_fact.price) as GMV 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3b74cea8/kylin-it/src/test/resources/query/sql_subquery/query19.sql

[15/20] kylin git commit: minor, fix wrong topn measure in sample cube

2017-09-23 Thread liyang

minor, fix wrong topn measure in sample cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56bb9720
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56bb9720
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56bb9720

Branch: refs/heads/master
Commit: 56bb972047f83f6f154347f57ed50082098af10c
Parents: 555617c
Author: Billy Liu 
Authored: Thu Sep 21 18:24:07 2017 +0800
Committer: SHAOFENG SHI 
Committed: Thu Sep 21 20:38:42 2017 +0800

--
 examples/sample_cube/template/cube_desc/kylin_sales_cube.json | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56bb9720/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 76fbb0a..1f16a5b 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -175,7 +175,11 @@
   "next_parameter" : null
 }
   },
-  "returntype" : "topn(100, 6)"
+  "returntype" : "topn(100)",
+  "configuration": {
+"topn.encoding.KYLIN_SALES.SELLER_ID": "dict",
+"topn.encoding_version.KYLIN_SALES.SELLER_ID": "1"
+  }
 }
   } ],
   "rowkey" : {

[04/20] kylin git commit: minor, enhance buildSingleColumnRangeCondition

2017-09-23 Thread liyang

minor, enhance buildSingleColumnRangeCondition


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58ca04ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58ca04ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58ca04ef

Branch: refs/heads/master
Commit: 58ca04efc1b10b38bd996513a41233f5a31c0180
Parents: 0d55c31
Author: Cheng Wang 
Authored: Mon Sep 18 21:11:05 2017 +0800
Committer: liyang-gmt8 
Committed: Mon Sep 18 21:26:44 2017 +0800

--
 .../apache/kylin/metadata/model/PartitionDesc.java |  8 +++-
 .../DefaultPartitionConditionBuilderTest.java  | 17 +
 2 files changed, 20 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/58ca04ef/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index cfa062b..dcbbfd2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -184,7 +184,7 @@ public class PartitionDesc implements Serializable {
 public String buildDateRangeCondition(PartitionDesc partDesc, 
SegmentRange segRange) {
 long startInclusive = (Long) segRange.start.v;
 long endExclusive = (Long) segRange.end.v;
-
+
 TblColRef partitionDateColumn = 
partDesc.getPartitionDateColumnRef();
 TblColRef partitionTimeColumn = 
partDesc.getPartitionTimeColumnRef();
 StringBuilder builder = new StringBuilder();
@@ -232,6 +232,12 @@ public class PartitionDesc implements Serializable {
 private static void buildSingleColumnRangeCondition(StringBuilder 
builder, TblColRef partitionColumn,
 long startInclusive, long endExclusive, String 
partitionColumnDateFormat) {
 String partitionColumnName = partitionColumn.getIdentity();
+
+if (endExclusive <= startInclusive) {
+builder.append("1=1");
+return;
+}
+
 if (startInclusive > 0) {
 builder.append(partitionColumnName + " >= '"
 + DateFormat.formatToDateStr(startInclusive, 
partitionColumnDateFormat) + "'");

http://git-wip-us.apache.org/repos/asf/kylin/blob/58ca04ef/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
 
b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
index 19d6f2f..cc7156a 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
@@ -54,7 +54,12 @@ public class DefaultPartitionConditionBuilderTest extends 
LocalFileMetadataTestC
 partitionDesc.setPartitionDateFormat("-MM-dd");
 TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"), 
DateFormat.stringToMillis("2016-02-23"));
 String condition = 
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, range);
-Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND 
UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition);
+Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND 
UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'",
+condition);
+
+range = new TSRange(0L, 0L);
+condition = 
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, range);
+Assert.assertEquals("1=1", condition);
 }
 
 @Test
@@ -64,7 +69,8 @@ public class DefaultPartitionConditionBuilderTest extends 
LocalFileMetadataTestC
 partitionDesc.setPartitionTimeColumnRef(col);
 partitionDesc.setPartitionTimeColumn(col.getCanonicalName());
 partitionDesc.setPartitionTimeFormat("HH");
-TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 
00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00"));
+TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 
00:00:00"),
+DateFormat.stringToMillis("2016-02-23 01:00:00"));
 String condition = 
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, range);

[14/20] kylin git commit: minor, add a switch to enable/disable Table ACL.

2017-09-23 Thread liyang

minor, add a switch to enable/disable Table ACL.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/555617cb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/555617cb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/555617cb

Branch: refs/heads/master
Commit: 555617cb07362dbe80e53fdfc317d641acc25e19
Parents: 940aef6
Author: tttMelody <245915...@qq.com>
Authored: Thu Sep 21 17:49:32 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Sep 21 17:57:08 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java  | 4 
 core-common/src/main/resources/kylin-defaults.properties| 3 ++-
 .../java/org/apache/kylin/query/security/QueryIntercept.java| 5 +
 .../java/org/apache/kylin/rest/security/TableIntercept.java | 5 +
 4 files changed, 16 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/555617cb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d535b7d..a86464f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1157,6 +1157,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.query.pushdown.jdbc.pool-min-idle", 
"0"));
 }
 
+public boolean isTableACLEnabled() {
+return 
Boolean.valueOf(this.getOptional("kylin.query.acl.table-acl-enabled", "true"));
+}
+
 // 

 // SERVER
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/555617cb/core-common/src/main/resources/kylin-defaults.properties
--
diff --git a/core-common/src/main/resources/kylin-defaults.properties 
b/core-common/src/main/resources/kylin-defaults.properties
index c2da3f8..bb03520 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -259,7 +259,7 @@ 
kylin.engine.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
 
 
#kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl
 
-#kylin.query.pushdown.update-enabled = false
+#kylin.query.pushdown.update-enabled=false
 #kylin.query.pushdown.jdbc.url=jdbc:hive2://sandbox:1/default
 #kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver
 #kylin.query.pushdown.jdbc.username=hive
@@ -270,4 +270,5 @@ 
kylin.engine.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
 #kylin.query.pushdown.jdbc.pool-min-idle=0
 
 ### TABLE ACL
+kylin.query.acl.table-acl-enabled=true
 kylin.query.intercepts=org.apache.kylin.rest.security.TableIntercept
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/555617cb/query/src/main/java/org/apache/kylin/query/security/QueryIntercept.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/security/QueryIntercept.java 
b/query/src/main/java/org/apache/kylin/query/security/QueryIntercept.java
index 8df0da7..2312568 100644
--- a/query/src/main/java/org/apache/kylin/query/security/QueryIntercept.java
+++ b/query/src/main/java/org/apache/kylin/query/security/QueryIntercept.java
@@ -26,6 +26,9 @@ import org.apache.kylin.query.relnode.OLAPContext;
 public abstract class QueryIntercept {
 
 public void intercept(List contexts) {
+if (!isEnabled()) {
+return;
+}
 Collection userIdentifierBlackList = 
getIdentifierBlackList(contexts);
 intercept(contexts, userIdentifierBlackList);
 }
@@ -43,6 +46,8 @@ public abstract class QueryIntercept {
 }
 }
 
+protected abstract boolean isEnabled();
+
 protected abstract Collection 
getQueryIdentifiers(List contexts);
 
 protected abstract Collection 
getIdentifierBlackList(List contexts);

http://git-wip-us.apache.org/repos/asf/kylin/blob/555617cb/server-base/src/main/java/org/apache/kylin/rest/security/TableIntercept.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/security/TableIntercept.java 
b/server-base/src/main/java/org/apache/kylin/rest/security/TableIntercept.java
index c87f674..442ce88 100644
---

[03/20] kylin git commit: Minor, add version info into sample cube

2017-09-23 Thread liyang

Minor, add version info into sample cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0d55c311
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0d55c311
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0d55c311

Branch: refs/heads/master
Commit: 0d55c3118aeea323fc7d33b05e46288628ab4ab3
Parents: 3177d79
Author: auphyroc99 <454530...@qq.com>
Authored: Mon Sep 18 18:12:22 2017 +0800
Committer: Roger Shi 
Committed: Mon Sep 18 19:02:01 2017 +0800

--
 build/bin/sample.sh   | 7 +++
 examples/sample_cube/template/cube_desc/kylin_sales_cube.json | 1 +
 .../sample_cube/template/cube_desc/kylin_streaming_cube.json  | 1 +
 3 files changed, 9 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0d55c311/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 9545362..ee4aa25 100755
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -88,6 +88,13 @@ cp -rf ${KYLIN_HOME}/sample_cube/template/* 
${KYLIN_HOME}/sample_cube/metadata
 sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 
+ Add version info into cubes
+kylin_version_str=`bash ${KYLIN_HOME}/bin/kylin.sh version | grep 
kylin.version`
+kylin_version=${kylin_version_str#*:}
+echo "kylin version is "$kylin_version
+sed -i "s/%default_version%/${kylin_version}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_version%/${kylin_version}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_streaming_cube.json
+
  Replace the 'DEFAULT' with kylin.source.hive.database-for-flat-table
 sed -i "s/DEFAULT./$sample_database./g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 sed -i "s/DEFAULT./$sample_database./g" 
${KYLIN_HOME}/sample_cube/metadata/model_desc/kylin_sales_model.json

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d55c311/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 34aa536..76fbb0a 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -1,6 +1,7 @@
 {
   "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
   "last_modified" : 1451468470824,
+  "version": "%default_version%",
   "name" : "kylin_sales_cube",
   "model_name" : "kylin_sales_model",
   "description" : null,

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d55c311/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
index ff7fb25..2bd5584 100644
--- a/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
@@ -1,5 +1,6 @@
 {
   "uuid" : "901ed15e-7769-4c66-b7ae-fbdc971cd092",
+  "version": "%default_version%",
   "name" : "kylin_streaming_cube",
   "description" : "",
   "dimensions" : [ {

[10/20] kylin git commit: minor, change the way that Table ACL get all tables.

2017-09-23 Thread liyang

minor, change the way that Table ACL get all tables.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9af6a3f0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9af6a3f0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9af6a3f0

Branch: refs/heads/master
Commit: 9af6a3f0cd2788b9df412c10c1ac237ff8b6d0ef
Parents: a7fd3b1
Author: tttMelody <245915...@qq.com>
Authored: Wed Sep 20 16:20:17 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Wed Sep 20 16:46:10 2017 +0800

--
 .../kylin/query/security/QueryInterceptUtil.java  | 13 +++--
 .../rest/security/QueryWithTableACLTest.java  | 18 +++---
 2 files changed, 22 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9af6a3f0/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java 
b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
index b6a97e5..fa094db 100644
--- 
a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
+++ 
b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.calcite.sql.SqlCall;
 import org.apache.calcite.sql.SqlIdentifier;
@@ -38,6 +39,7 @@ import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.model.tool.CalciteParser;
 import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.relnode.OLAPTableScan;
 
 import com.google.common.base.Preconditions;
 
@@ -66,7 +68,7 @@ public class QueryInterceptUtil {
 
 public static Set getAllColsWithTblAndSchema(String project, 
List contexts) {
 // all columns with table and DB. Like DB.TABLE.COLUMN
-Set allColWithTblAndSchema = new HashSet<>();
+Set allColWithTblAndSchema = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 
 for (OLAPContext context : contexts) {
 for (TblColRef tblColRef : context.allColumns) {
@@ -156,12 +158,11 @@ public class QueryInterceptUtil {
 }
 
 public static Set getAllTblsWithSchema(List contexts) 
{
-// all tables with DB, Like DB.TABLE, may have same table, so use set.
-Set tableWithSchema = new HashSet<>();
+// all tables with DB, Like DB.TABLE
+Set tableWithSchema = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 for (OLAPContext context : contexts) {
-Set allColumns = context.allColumns;
-for (TblColRef tblColRef : allColumns) {
-tableWithSchema.add(tblColRef.getTableWithSchema());
+for (OLAPTableScan tableScan : context.allTableScans) {
+
tableWithSchema.add(tableScan.getTableRef().getTableIdentity());
 }
 }
 return tableWithSchema;

http://git-wip-us.apache.org/repos/asf/kylin/blob/9af6a3f0/server/src/test/java/org/apache/kylin/rest/security/QueryWithTableACLTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/security/QueryWithTableACLTest.java
 
b/server/src/test/java/org/apache/kylin/rest/security/QueryWithTableACLTest.java
index 1f3539e..950f586 100644
--- 
a/server/src/test/java/org/apache/kylin/rest/security/QueryWithTableACLTest.java
+++ 
b/server/src/test/java/org/apache/kylin/rest/security/QueryWithTableACLTest.java
@@ -18,6 +18,9 @@
 
 package org.apache.kylin.rest.security;
 
+import java.io.IOException;
+import java.sql.SQLException;
+
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.metadata.acl.TableACLManager;
@@ -30,9 +33,6 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
-import java.io.IOException;
-import java.sql.SQLException;
-
 public class QueryWithTableACLTest extends LocalFileMetadataTestCase {
 private static final String PROJECT = "DEFAULT";
 private static final String ADMIN = "ADMIN";
@@ -65,6 +65,18 @@ public class QueryWithTableACLTest extends 
LocalFileMetadataTestCase {
 QuerACLTestUtil.mockQuery(PROJECT, "select * from STREAMING_TABLE");
 }
 
+@Test
+public void testFailQueryWithCountStar() throws SQLException, IOException {
+QuerACLTestUtil.setUser(MODELER);
+QuerACLTestUtil.mockQuery(PROJECT, "select count(*) from 
STREAMING_TABLE");
+
+QuerACLTestUtil.setUser(ADMIN);
+

[01/20] kylin git commit: minor, remove unused code

2017-09-23 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 658353ad5 -> 0a8bdc27a


minor, remove unused code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4ad6f08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4ad6f08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4ad6f08

Branch: refs/heads/master
Commit: b4ad6f084cad7e0aca8d50301d991e7802c0cdcf
Parents: bf0e2e1
Author: lidongsjtu 
Authored: Mon Sep 18 14:12:14 2017 +0800
Committer: Dong Li 
Committed: Mon Sep 18 15:19:16 2017 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeInstance.java | 10 --
 1 file changed, 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b4ad6f08/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 7d539c6..ac41970 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -404,7 +404,6 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 @Override
 public CapabilityResult isCapable(SQLDigest digest) {
 CapabilityResult result = CubeCapabilityChecker.check(this, digest);
-result = localCapacityCheck(digest, result);
 if (result.capable) {
 result.cost = getCost(digest);
 for (CapabilityInfluence i : result.influences) {
@@ -416,15 +415,6 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return result;
 }
 
-private CapabilityResult localCapacityCheck(SQLDigest digest, 
CapabilityResult originResult) {
-if (this.getDescriptor().getConfig().isDisableCubeNoAggSQL()) {
-CapabilityResult notCap = new CapabilityResult();
-notCap.capable = false;
-return digest.aggregations.isEmpty() ? notCap : originResult ;
-}
-return originResult;
-}
-
 public int getCost(SQLDigest digest) {
 int calculatedCost = cost;

[06/20] kylin git commit: minor, make that project management can not load/unload tables.

2017-09-23 Thread liyang

minor, make that project management can not load/unload tables.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8f3f459e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8f3f459e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8f3f459e

Branch: refs/heads/master
Commit: 8f3f459e35728b9238c36e868164be123b102d69
Parents: ea92edf
Author: tttMelody <245915...@qq.com>
Authored: Tue Sep 19 21:01:34 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Sep 19 21:21:12 2017 +0800

--
 .../org/apache/kylin/rest/service/KafkaConfigService.java |  6 +++---
 .../org/apache/kylin/rest/service/StreamingService.java   |  6 +++---
 .../java/org/apache/kylin/rest/service/TableService.java  | 10 +-
 3 files changed, 11 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3f459e/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java
 
b/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java
index 3983a30..260cbc0 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java
@@ -69,7 +69,7 @@ public class KafkaConfigService extends BasicService {
 }
 
 public KafkaConfig createKafkaConfig(KafkaConfig config, String project) 
throws IOException {
-aclEvaluate.checkProjectWritePermission(project);
+aclEvaluate.checkProjectAdminPermission(project);
 Message msg = MsgPicker.getMsg();
 
 if (getKafkaManager().getKafkaConfig(config.getName()) != null) {
@@ -80,7 +80,7 @@ public class KafkaConfigService extends BasicService {
 }
 
 public KafkaConfig updateKafkaConfig(KafkaConfig config, String project) 
throws IOException {
-aclEvaluate.checkProjectWritePermission(project);
+aclEvaluate.checkProjectAdminPermission(project);
 return getKafkaManager().updateKafkaConfig(config);
 }
 
@@ -90,7 +90,7 @@ public class KafkaConfigService extends BasicService {
 }
 
 public void dropKafkaConfig(KafkaConfig config, String project) throws 
IOException {
-aclEvaluate.checkProjectWritePermission(project);
+aclEvaluate.checkProjectAdminPermission(project);
 getKafkaManager().removeKafkaConfig(config);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3f459e/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java
index 2871285..1f907f8 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/service/StreamingService.java
@@ -67,7 +67,7 @@ public class StreamingService extends BasicService {
 }
 
 public StreamingConfig createStreamingConfig(StreamingConfig config, 
String project) throws IOException {
-aclEvaluate.checkProjectWritePermission(project);
+aclEvaluate.checkProjectAdminPermission(project);
 Message msg = MsgPicker.getMsg();
 
 if (getStreamingManager().getStreamingConfig(config.getName()) != 
null) {
@@ -78,12 +78,12 @@ public class StreamingService extends BasicService {
 }
 
 public StreamingConfig updateStreamingConfig(StreamingConfig config, 
String project) throws IOException {
-aclEvaluate.checkProjectWritePermission(project);
+aclEvaluate.checkProjectAdminPermission(project);
 return getStreamingManager().updateStreamingConfig(config);
 }
 
 public void dropStreamingConfig(StreamingConfig config, String project) 
throws IOException {
-aclEvaluate.checkProjectWritePermission(project);
+aclEvaluate.checkProjectAdminPermission(project);
 getStreamingManager().removeStreamingConfig(config);
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3f459e/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index 4680c47..a7c851c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -114,7

1 2 3 4 5 6 7 8 9 10 >

1 - 100 of 1436 matches

Mail list logo