Build failed in Jenkins: Kylin-Master-JDK-1.7 #943

[Apache Jenkins Server]

See 

--
Started by timer
[EnvInject] - Loading node environment variables.
Building remotely on H34 (ubuntu xenial) in workspace 

 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url 
 > https://git-wip-us.apache.org/repos/asf/kylin.git # timeout=10
Fetching upstream changes from https://git-wip-us.apache.org/repos/asf/kylin.git
 > git --version # timeout=10
 > git fetch --tags --progress 
 > https://git-wip-us.apache.org/repos/asf/kylin.git 
 > +refs/heads/*:refs/remotes/origin/*
 > git rev-parse refs/remotes/origin/master^{commit} # timeout=10
 > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10
Checking out Revision edc4397aa1f168a75a612639376bff89d6424310 
(refs/remotes/origin/master)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f edc4397aa1f168a75a612639376bff89d6424310
Commit message: "minor, add comments in CubeInstance"
 > git rev-list edc4397aa1f168a75a612639376bff89d6424310 # timeout=10
Parsing POMs
Established TCP socket on 33608
maven32-agent.jar already up to date
maven32-interceptor.jar already up to date
maven3-interceptor-commons.jar already up to date
[Kylin-Master-JDK-1.7] $ /home/jenkins/tools/java/latest1.7/bin/java -Xmx2g 
-Xms256m -cp 
/home/jenkins/jenkins-slave/maven32-agent.jar:/home/jenkins/tools/maven/apache-maven-3.2.5/boot/plexus-classworlds-2.5.2.jar:/home/jenkins/tools/maven/apache-maven-3.2.5/conf/logging
 jenkins.maven3.agent.Maven32Main /home/jenkins/tools/maven/apache-maven-3.2.5 
/home/jenkins/jenkins-slave/slave.jar 
/home/jenkins/jenkins-slave/maven32-interceptor.jar 
/home/jenkins/jenkins-slave/maven3-interceptor-commons.jar 33608
Exception in thread "main" java.lang.UnsupportedClassVersionError: 
hudson/remoting/Launcher : Unsupported major.minor version 52.0
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at 
java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.loadClassFromSelf(ClassRealm.java:401)
at 
org.codehaus.plexus.classworlds.strategy.SelfFirstStrategy.loadClass(SelfFirstStrategy.java:42)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.unsynchronizedLoadClass(ClassRealm.java:271)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.loadClass(ClassRealm.java:247)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.loadClass(ClassRealm.java:239)
at jenkins.maven3.agent.Maven32Main.main(Maven32Main.java:143)
at jenkins.maven3.agent.Maven32Main.main(Maven32Main.java:74)
ERROR: Failed to parse POMs
java.io.EOFException: unexpected stream termination
at hudson.remoting.ChannelBuilder.negotiate(ChannelBuilder.java:408)
at hudson.remoting.ChannelBuilder.build(ChannelBuilder.java:353)
at hudson.slaves.Channels.forProcess(Channels.java:115)
at 
hudson.maven.AbstractMavenProcessFactory.newProcess(AbstractMavenProcessFactory.java:294)
at hudson.maven.ProcessCache.get(ProcessCache.java:236)
at 
hudson.maven.MavenModuleSetBuild$MavenModuleSetBuildExecution.doRun(MavenModuleSetBuild.java:804)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:504)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:421)
Not sending mail to unregistered user 100650...@qq.com
Not sending mail to unregistered user 245915...@qq.com
Not sending mail to unregistered user 550175...@qq.com
Not sending mail to unregistered user cheng.w...@kyligence.io
Not sending mail to unregistered user chunen...@kyligence.io
Not sending mail to unregistered user event.dim...@gmail.com
Not sending mail to unregistered user git...@alasdairhodge.co.uk
Not sending mail to unregistered user liuyiming@gmail.com
Not sending mail to unregistered user m...@kyligence.io
Not sending mail to unregistered user rogershijich...@gmail.com
Not sending mail to unregistered user sheng.zh...@kyligence.io
Not sending mail to unregistered user zjsy...@sjtu.org

Build failed in Jenkins: Kylin-Master-JDK-1.8 #939

[Apache Jenkins Server]

See 


Changes:

[liyang] minor, add comments in CubeInstance

--
[...truncated 39.28 KB...]
[INFO] 
[INFO] --- maven-dependency-plugin:2.10:copy (copy-jamm) @ kylin-jdbc ---
[INFO] Configured Artifact: com.github.jbellis:jamm:?:jar
[INFO] Copying jamm-0.3.1.jar to 

[INFO] 
[INFO] --- maven-resources-plugin:3.0.1:testResources (default-testResources) @ 
kylin-jdbc ---
[INFO] Using 'UTF-8' encoding to copy filtered resources.
[INFO] skip non existing resourceDirectory 

[INFO] Copying 3 resources
[INFO] 
[INFO] --- maven-compiler-plugin:3.5.1:testCompile (default-testCompile) @ 
kylin-jdbc ---
[INFO] Changes detected - recompiling the module!
[INFO] Compiling 5 source files to 

[INFO] 
[INFO] --- maven-surefire-plugin:2.19.1:test (default-test) @ kylin-jdbc ---
[INFO] Surefire report directory: 


---
 T E S T S
---
Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=512M; 
support was removed in 8.0
Running org.apache.kylin.jdbc.SQLResonseStubTest
Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.917 sec - in 
org.apache.kylin.jdbc.SQLResonseStubTest
Running org.apache.kylin.jdbc.DriverTest
2017-12-31 23:59:47,227 DEBUG [main] jdbc.KylinConnection:62 : Kylin base url 
test_url, project name test_db
2017-12-31 23:59:47,366 DEBUG [main] jdbc.KylinConnection:62 : Kylin base url 
test_url, project name test_db
2017-12-31 23:59:47,369 DEBUG [main] jdbc.KylinConnection:62 : Kylin base url 
test_url, project name test_db
2017-12-31 23:59:47,372 DEBUG [main] jdbc.KylinConnection:62 : Kylin base url 
test_url, project name test_db
Tests run: 7, Failures: 0, Errors: 0, Skipped: 2, Time elapsed: 0.533 sec - in 
org.apache.kylin.jdbc.DriverTest

Results :

Tests run: 9, Failures: 0, Errors: 0, Skipped: 2

[JENKINS] Recording test results
[INFO] 
[INFO] --- jacoco-maven-plugin:0.7.8:report (post-test) @ kylin-jdbc ---
[INFO] Skipping JaCoCo execution due to missing execution data file.
[PMD] No report found for mojo report
[INFO] 
[INFO] 
[INFO] Skipping Apache Kylin - Assembly
[INFO] This project has been banned from the build due to previous failures.
[INFO] 
[INFO] 
[INFO] 
[INFO] Skipping Apache Kylin - Tool
[INFO] This project has been banned from the build due to previous failures.
[INFO] 
[INFO] 
[INFO] 
[INFO] Skipping Apache Kylin - Tool Assembly
[INFO] This project has been banned from the build due to previous failures.
[INFO] 
[INFO] 
[INFO] 
[INFO] Skipping Apache Kylin - Integration Test
[INFO] This project has been banned from the build due to previous failures.
[INFO] 
[INFO] 
[INFO] 
[INFO] Building Apache Kylin - Tomcat Extension 2.3.0-SNAPSHOT
[INFO] 
[INFO] 
[INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ kylin-tomcat-ext ---
[INFO] Deleting 

[INFO] 
[INFO] --- maven-checkstyle-plugin:2.17:check (check-style) @ kylin-tomcat-ext 
---
[INFO] Starting audit...
Audit done.
[CHECKSTYLE] Parsing file 

[CHECKSTYLE] Successfully parsed file 

 of module Apache Kylin - Tomcat Extension with 0 unique warnings and 0 
duplicates.
[CHECKSTYLE] Computing warning deltas based on reference build 

Build failed in Jenkins: Kylin-Master-JDK-1.7 #942

[Apache Jenkins Server]

See 


Changes:

[liyang] minor, add comments in CubeInstance

--
Started by an SCM change
[EnvInject] - Loading node environment variables.
Building remotely on H34 (ubuntu xenial) in workspace 

 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url 
 > https://git-wip-us.apache.org/repos/asf/kylin.git # timeout=10
Fetching upstream changes from https://git-wip-us.apache.org/repos/asf/kylin.git
 > git --version # timeout=10
 > git fetch --tags --progress 
 > https://git-wip-us.apache.org/repos/asf/kylin.git 
 > +refs/heads/*:refs/remotes/origin/*
 > git rev-parse refs/remotes/origin/master^{commit} # timeout=10
 > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10
Checking out Revision edc4397aa1f168a75a612639376bff89d6424310 
(refs/remotes/origin/master)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f edc4397aa1f168a75a612639376bff89d6424310
Commit message: "minor, add comments in CubeInstance"
 > git rev-list 6f4dbc1210ad77578f37fcb21a9acd6024571765 # timeout=10
Parsing POMs
Established TCP socket on 39041
maven32-agent.jar already up to date
maven32-interceptor.jar already up to date
maven3-interceptor-commons.jar already up to date
[Kylin-Master-JDK-1.7] $ /home/jenkins/tools/java/latest1.7/bin/java -Xmx2g 
-Xms256m -cp 
/home/jenkins/jenkins-slave/maven32-agent.jar:/home/jenkins/tools/maven/apache-maven-3.2.5/boot/plexus-classworlds-2.5.2.jar:/home/jenkins/tools/maven/apache-maven-3.2.5/conf/logging
 jenkins.maven3.agent.Maven32Main /home/jenkins/tools/maven/apache-maven-3.2.5 
/home/jenkins/jenkins-slave/slave.jar 
/home/jenkins/jenkins-slave/maven32-interceptor.jar 
/home/jenkins/jenkins-slave/maven3-interceptor-commons.jar 39041
Exception in thread "main" java.lang.UnsupportedClassVersionError: 
hudson/remoting/Launcher : Unsupported major.minor version 52.0
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at 
java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.loadClassFromSelf(ClassRealm.java:401)
at 
org.codehaus.plexus.classworlds.strategy.SelfFirstStrategy.loadClass(SelfFirstStrategy.java:42)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.unsynchronizedLoadClass(ClassRealm.java:271)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.loadClass(ClassRealm.java:247)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.loadClass(ClassRealm.java:239)
at jenkins.maven3.agent.Maven32Main.main(Maven32Main.java:143)
at jenkins.maven3.agent.Maven32Main.main(Maven32Main.java:74)
ERROR: Failed to parse POMs
java.io.EOFException: unexpected stream termination
at hudson.remoting.ChannelBuilder.negotiate(ChannelBuilder.java:408)
at hudson.remoting.ChannelBuilder.build(ChannelBuilder.java:353)
at hudson.slaves.Channels.forProcess(Channels.java:115)
at 
hudson.maven.AbstractMavenProcessFactory.newProcess(AbstractMavenProcessFactory.java:294)
at hudson.maven.ProcessCache.get(ProcessCache.java:236)
at 
hudson.maven.MavenModuleSetBuild$MavenModuleSetBuildExecution.doRun(MavenModuleSetBuild.java:804)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:504)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:421)
Not sending mail to unregistered user 100650...@qq.com
Not sending mail to unregistered user 245915...@qq.com
Not sending mail to unregistered user 550175...@qq.com
Not sending mail to unregistered user cheng.w...@kyligence.io
Not sending mail to unregistered user chunen...@kyligence.io
Not sending mail to unregistered user event.dim...@gmail.com
Not sending mail to unregistered user git...@alasdairhodge.co.uk
Not sending mail to unregistered user liuyiming@gmail.com
Not sending mail to unregistered user m...@kyligence.io
Not sending mail to unregistered user rogershijich...@gmail.com
Not sending mail to unregistered user sheng.zh...@kyligence.io
Not sending mail to unregistered user zjsy...@sjtu.org

kylin git commit: minor, add comments in CubeInstance

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 6f4dbc121 -> edc4397aa


minor, add comments in CubeInstance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/edc4397a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/edc4397a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/edc4397a

Branch: refs/heads/master
Commit: edc4397aa1f168a75a612639376bff89d6424310
Parents: 6f4dbc1
Author: Li Yang 
Authored: Mon Jan 1 07:57:10 2018 +0800
Committer: Li Yang 
Committed: Mon Jan 1 07:57:10 2018 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/edc4397a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 79341cb..a299203 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -578,6 +578,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 }
 
+// For JSON serialization of this attribute, use CubeInstanceResponse
 public long getSizeKB() {
 long sizeKb = 0L;
 
@@ -588,6 +589,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return sizeKb;
 }
 
+// For JSON serialization of this attribute, use CubeInstanceResponse
 public long getInputRecordCount() {
 long sizeRecordCount = 0L;
 
@@ -598,6 +600,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return sizeRecordCount;
 }
 
+// For JSON serialization of this attribute, use CubeInstanceResponse
 public long getInputRecordSizeMB() {
 long sizeRecordSize = 0L;
 

Build failed in Jenkins: Kylin-Master-JDK-1.8 #938

[Apache Jenkins Server]

See 


Changes:

[liyang] KYLIN-2723 Rename config kylin.core.metrics.xxx to kylin.metrics.xxx 
for

--
[...truncated 38.78 KB...]
[INFO] 
[INFO] --- maven-dependency-plugin:2.10:copy (copy-jamm) @ kylin-jdbc ---
[INFO] Configured Artifact: com.github.jbellis:jamm:?:jar
[INFO] Copying jamm-0.3.1.jar to 

[INFO] 
[INFO] --- maven-resources-plugin:3.0.1:testResources (default-testResources) @ 
kylin-jdbc ---
[INFO] Using 'UTF-8' encoding to copy filtered resources.
[INFO] skip non existing resourceDirectory 

[INFO] Copying 3 resources
[INFO] 
[INFO] --- maven-compiler-plugin:3.5.1:testCompile (default-testCompile) @ 
kylin-jdbc ---
[INFO] Changes detected - recompiling the module!
[INFO] Compiling 5 source files to 

[INFO] 
[INFO] --- maven-surefire-plugin:2.19.1:test (default-test) @ kylin-jdbc ---
[INFO] Surefire report directory: 


---
 T E S T S
---
Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=512M; 
support was removed in 8.0
Running org.apache.kylin.jdbc.SQLResonseStubTest
Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.887 sec - in 
org.apache.kylin.jdbc.SQLResonseStubTest
Running org.apache.kylin.jdbc.DriverTest
2017-12-31 23:38:01,235 DEBUG [main] jdbc.KylinConnection:62 : Kylin base url 
test_url, project name test_db
2017-12-31 23:38:01,326 DEBUG [main] jdbc.KylinConnection:62 : Kylin base url 
test_url, project name test_db
2017-12-31 23:38:01,328 DEBUG [main] jdbc.KylinConnection:62 : Kylin base url 
test_url, project name test_db
2017-12-31 23:38:01,332 DEBUG [main] jdbc.KylinConnection:62 : Kylin base url 
test_url, project name test_db
Tests run: 7, Failures: 0, Errors: 0, Skipped: 2, Time elapsed: 0.6 sec - in 
org.apache.kylin.jdbc.DriverTest

Results :

Tests run: 9, Failures: 0, Errors: 0, Skipped: 2

[JENKINS] Recording test results
[INFO] 
[INFO] --- jacoco-maven-plugin:0.7.8:report (post-test) @ kylin-jdbc ---
[INFO] Skipping JaCoCo execution due to missing execution data file.
[PMD] No report found for mojo report
[INFO] 
[INFO] 
[INFO] Skipping Apache Kylin - Assembly
[INFO] This project has been banned from the build due to previous failures.
[INFO] 
[INFO] 
[INFO] 
[INFO] Skipping Apache Kylin - Tool
[INFO] This project has been banned from the build due to previous failures.
[INFO] 
[INFO] 
[INFO] 
[INFO] Skipping Apache Kylin - Tool Assembly
[INFO] This project has been banned from the build due to previous failures.
[INFO] 
[INFO] 
[INFO] 
[INFO] Skipping Apache Kylin - Integration Test
[INFO] This project has been banned from the build due to previous failures.
[INFO] 
[INFO] 
[INFO] 
[INFO] Building Apache Kylin - Tomcat Extension 2.3.0-SNAPSHOT
[INFO] 
[INFO] 
[INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ kylin-tomcat-ext ---
[INFO] Deleting 

[INFO] 
[INFO] --- maven-checkstyle-plugin:2.17:check (check-style) @ kylin-tomcat-ext 
---
[INFO] Starting audit...
Audit done.
[CHECKSTYLE] Parsing file 

[CHECKSTYLE] Successfully parsed file 

 of module Apache Kylin - Tomcat Extension with 0 unique warnings and 0 
duplicates.
[CHECKSTYLE] Computing 

Build failed in Jenkins: Kylin-Master-JDK-1.7 #941

[Apache Jenkins Server]

See 


Changes:

[liyang] KYLIN-2723 Rename config kylin.core.metrics.xxx to kylin.metrics.xxx 
for

--
Started by an SCM change
[EnvInject] - Loading node environment variables.
Building remotely on H34 (ubuntu xenial) in workspace 

 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url 
 > https://git-wip-us.apache.org/repos/asf/kylin.git # timeout=10
Fetching upstream changes from https://git-wip-us.apache.org/repos/asf/kylin.git
 > git --version # timeout=10
 > git fetch --tags --progress 
 > https://git-wip-us.apache.org/repos/asf/kylin.git 
 > +refs/heads/*:refs/remotes/origin/*
 > git rev-parse refs/remotes/origin/master^{commit} # timeout=10
 > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10
Checking out Revision 6f4dbc1210ad77578f37fcb21a9acd6024571765 
(refs/remotes/origin/master)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f 6f4dbc1210ad77578f37fcb21a9acd6024571765
Commit message: "KYLIN-2723 Rename config kylin.core.metrics.xxx to 
kylin.metrics.xxx for consistency."
 > git rev-list 4d50b26972bb7bbaff852172990e0f189f987673 # timeout=10
Parsing POMs
Established TCP socket on 42608
maven32-agent.jar already up to date
maven32-interceptor.jar already up to date
maven3-interceptor-commons.jar already up to date
[Kylin-Master-JDK-1.7] $ /home/jenkins/tools/java/latest1.7/bin/java -Xmx2g 
-Xms256m -cp 
/home/jenkins/jenkins-slave/maven32-agent.jar:/home/jenkins/tools/maven/apache-maven-3.2.5/boot/plexus-classworlds-2.5.2.jar:/home/jenkins/tools/maven/apache-maven-3.2.5/conf/logging
 jenkins.maven3.agent.Maven32Main /home/jenkins/tools/maven/apache-maven-3.2.5 
/home/jenkins/jenkins-slave/slave.jar 
/home/jenkins/jenkins-slave/maven32-interceptor.jar 
/home/jenkins/jenkins-slave/maven3-interceptor-commons.jar 42608
Exception in thread "main" java.lang.UnsupportedClassVersionError: 
hudson/remoting/Launcher : Unsupported major.minor version 52.0
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at 
java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.loadClassFromSelf(ClassRealm.java:401)
at 
org.codehaus.plexus.classworlds.strategy.SelfFirstStrategy.loadClass(SelfFirstStrategy.java:42)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.unsynchronizedLoadClass(ClassRealm.java:271)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.loadClass(ClassRealm.java:247)
at 
org.codehaus.plexus.classworlds.realm.ClassRealm.loadClass(ClassRealm.java:239)
at jenkins.maven3.agent.Maven32Main.main(Maven32Main.java:143)
at jenkins.maven3.agent.Maven32Main.main(Maven32Main.java:74)
ERROR: Failed to parse POMs
java.io.EOFException: unexpected stream termination
at hudson.remoting.ChannelBuilder.negotiate(ChannelBuilder.java:408)
at hudson.remoting.ChannelBuilder.build(ChannelBuilder.java:353)
at hudson.slaves.Channels.forProcess(Channels.java:115)
at 
hudson.maven.AbstractMavenProcessFactory.newProcess(AbstractMavenProcessFactory.java:294)
at hudson.maven.ProcessCache.get(ProcessCache.java:236)
at 
hudson.maven.MavenModuleSetBuild$MavenModuleSetBuildExecution.doRun(MavenModuleSetBuild.java:804)
at 
hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:504)
at hudson.model.Run.execute(Run.java:1724)
at hudson.maven.MavenModuleSetBuild.run(MavenModuleSetBuild.java:543)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:421)
Not sending mail to unregistered user 100650...@qq.com
Not sending mail to unregistered user 245915...@qq.com
Not sending mail to unregistered user 550175...@qq.com
Not sending mail to unregistered user cheng.w...@kyligence.io
Not sending mail to unregistered user chunen...@kyligence.io
Not sending mail to unregistered user event.dim...@gmail.com
Not sending mail to unregistered user git...@alasdairhodge.co.uk
Not sending mail to unregistered user liuyiming@gmail.com
Not sending mail to unregistered user m...@kyligence.io
Not sending mail to unregistered user rogershijich...@gmail.com
Not sending mail to 

kylin git commit: KYLIN-2723 Rename config kylin.core.metrics.xxx to kylin.metrics.xxx for consistency.

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 4d50b2697 -> 6f4dbc121


KYLIN-2723 Rename config kylin.core.metrics.xxx to kylin.metrics.xxx for 
consistency.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f4dbc12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f4dbc12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f4dbc12

Branch: refs/heads/master
Commit: 6f4dbc1210ad77578f37fcb21a9acd6024571765
Parents: 4d50b26
Author: Li Yang 
Authored: Mon Jan 1 07:35:11 2018 +0800
Committer: Li Yang 
Committed: Mon Jan 1 07:35:11 2018 +0800

--
 .../apache/kylin/common/KylinConfigBase.java| 22 ++--
 .../kylin/rest/metrics/QueryMetricsTest.java|  2 +-
 2 files changed, 12 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6f4dbc12/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2effdfb..667eea9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1516,28 +1516,28 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public boolean isKylinMetricsMonitorEnabled() {
-return 
Boolean.parseBoolean(getOptional("kylin.core.metrics.monitor-enabled", 
"false"));
+return 
Boolean.parseBoolean(getOptional("kylin.metrics.monitor-enabled", "false"));
 }
 
 public boolean isKylinMetricsReporterForQueryEnabled() {
-return 
Boolean.parseBoolean(getOptional("kylin.core.metrics.reporter-query-enabled", 
"false"));
+return 
Boolean.parseBoolean(getOptional("kylin.metrics.reporter-query-enabled", 
"false"));
 }
 
 public boolean isKylinMetricsReporterForJobEnabled() {
-return 
Boolean.parseBoolean(getOptional("kylin.core.metrics.reporter-job-enabled", 
"false"));
+return 
Boolean.parseBoolean(getOptional("kylin.metrics.reporter-job-enabled", 
"false"));
 }
 
 public String getKylinMetricsPrefix() {
-return getOptional("kylin.core.metrics.prefix", "KYLIN").toUpperCase();
+return getOptional("kylin.metrics.prefix", "KYLIN").toUpperCase();
 }
 
 public String getKylinMetricsActiveReservoirDefaultClass() {
-return getOptional("kylin.core.metrics.active-reservoir-default-class",
+return getOptional("kylin.metrics.active-reservoir-default-class",
 "org.apache.kylin.metrics.lib.impl.StubReservoir");
 }
 
 public String getKylinSystemCubeSinkDefaultClass() {
-return getOptional("kylin.core.metrics.system-cube-sink-default-class",
+return getOptional("kylin.metrics.system-cube-sink-default-class",
 "org.apache.kylin.metrics.lib.impl.hive.HiveSink");
 }
 
@@ -1546,25 +1546,25 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public String getKylinMetricsSubjectJob() {
-return getOptional("kylin.core.metrics.subject-job", "METRICS_JOB") + 
"_" + getKylinMetricsSubjectSuffix();
+return getOptional("kylin.metrics.subject-job", "METRICS_JOB") + "_" + 
getKylinMetricsSubjectSuffix();
 }
 
 public String getKylinMetricsSubjectJobException() {
-return getOptional("kylin.core.metrics.subject-job-exception", 
"METRICS_JOB_EXCEPTION") + "_"
+return getOptional("kylin.metrics.subject-job-exception", 
"METRICS_JOB_EXCEPTION") + "_"
 + getKylinMetricsSubjectSuffix();
 }
 
 public String getKylinMetricsSubjectQuery() {
-return getOptional("kylin.core.metrics.subject-query", 
"METRICS_QUERY") + "_" + getKylinMetricsSubjectSuffix();
+return getOptional("kylin.metrics.subject-query", "METRICS_QUERY") + 
"_" + getKylinMetricsSubjectSuffix();
 }
 
 public String getKylinMetricsSubjectQueryCube() {
-return getOptional("kylin.core.metrics.subject-query-cube", 
"METRICS_QUERY_CUBE") + "_"
+return getOptional("kylin.metrics.subject-query-cube", 
"METRICS_QUERY_CUBE") + "_"
 + getKylinMetricsSubjectSuffix();
 }
 
 public String getKylinMetricsSubjectQueryRpcCall() {
-return getOptional("kylin.core.metrics.subject-query-rpc", 
"METRICS_QUERY_RPC") + "_"
+return getOptional("kylin.metrics.subject-query-rpc", 
"METRICS_QUERY_RPC") + "_"
 + getKylinMetricsSubjectSuffix();
 }
 }

[kylin] Git Push Summary

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/tmp [deleted] 8cdc0e3cb

Build failed in Jenkins: Kylin-Master-JDK-1.8 #937

[Apache Jenkins Server]

See 


Changes:

[event.dimlas] Minor: do not hardcode DefaultQueryTransformer sequence.

[event.dimlas] Minor: Fix UT

[ma] Hide columns begin with "_KY_" for prepare queries

[liyang] minor, add project to cube list api

[github] minor, fix UT

[chen] kylin access for cube and model (#2782)

[github] KYLIN-2775 fix non-default database issue in sample model/cube (#2791)

[github] minor update KafkaSource log

[chen] remove access condition of modeler (#2798)

[zjsynce] Throw exception from pushdown instead of exception from calcite

[liyang] minor, append context info in Rel writer

[liyang] KYLIN-2930 make tablescan phy type always ARRAY

[chen] KYLIN-2756 Let 'LIMIT' be optional in 'inspect' page (#2813)

[lidong] minor, support real admin ldap-group name

[github] KYLIN-2945 global dict specific info change to use relative path 
(#2790)

[zjsynce] Shade commons-dbcp library in kylin-job jar

[liyang] fix old spark property in kylin.properties

[liyang] KYLIN-2944 HLLCSerializer, RawSerializer, PercentileSerializer returns

[liyang] KYLIN-2600 fix Incorrectly setting the range start

[liyang] minor, fix union query for ci

[rogershijicheng] KYLIN-2945 fix CI

[liyang] refactor, move HDFSResourceStore to the right module and others

[chen]  minor,kylin acl debug (#2874)

[zjsynce] Support function with name percentile_approx

[ma] minor, refine CubeMigrationCLI

[zjsynce] Shade commons-pool and commons-jocl library in kylin-job jar

[liyang] minor, refine TableExtDesc

[liyang] minor, export correct kylinConfig in ProjectInstance

[100650920] Revert "KYLIN-2777 Should throw error when sql contains non-equi 
left

[100650920] KYLIN-2777  Add test cases

[ma] minor, refine DiagnosisService

[550175214] minor, fix bug (#2908)

[event.dimlas] #2857 need pre processing in HivePushDownConverter

[liyang] Minor, set setCubeInstance func public.

[liyang] Return update date of cube response to the cube desc's update time

[liyang] Order models with data models' modified times

[liyang] minor refactor on PartitionDesc, make sure start time condition is

[liyang] bug fix in SegmentGTStartAndEnd

[zjsynce] Add function for doing pushdown query directly in KylinTestBase

[lidong] minor, update calcite dependency

[liyang] minor, fix log message in Broadcaster

[liyang] Build not a cube without date partition column.

[event.dimlas] Move out ComputedColumnDesc

[liyang] move NoRealizationFoundException RoutingIndicatorException

[liyang] KYLIN-2985 Cache temp json file created by each Calcite Connection

[liyang] KYLIN-2982 Avoid upgrade column in OLAPTable

[liyang] KYLIN-2982 update sum measure default return type

[github] minor, simplify OLAPSchemaFactory log

[shaofengshi] KYLIN-2991 fix NumberFormatException if partitionDateFormat is 
like

[lidong] minor, updates on OLAPContext

[shaofengshi] KYLIN-2797 Remove MR engine V1

[shaofengshi] KYLIN-2796 Remove the legacy "statisticsenabled" codes

[event.dimlas] #3062 pre-process MIN(1) and MAX(1)

[rogershijicheng] minor, make calcite rule configurable

[zjsynce] Do not do update query in push down engine for prepare query

[zjsynce] Add "limit 1" to the end of prepare sql of pushdown, to make it speed 
up

[liuyiming.vip] minor, drop invalid kylin.job.step.timeout in kylin.properties

[liuyiming.vip] KYLIN-1319, drop deprecated HadoopStatusChecker

[cheng.wang] Revert "Add "limit 1" to the end of prepare sql of pushdown, to 
make it

[cheng.wang] Revert "Do not do update query in push down engine for prepare 
query"

[ma] minor refactor

[liyang] KYLIN-2794 MultipleDictionaryValueEnumerator consider value order based

[rogershijicheng] KYLIN-3036 Allow complex column type when loading source table

[github] KYLIN-3038 cannot support sum of type-converted column SQL (#3124)

[liyang] minor, rename percentile to percentile_approx

[lidong] KYLIN-3040 fix segment name when refreshing non-partitioned cube

[zjsynce] Do not do update query in push down engine for prepare query

[zjsynce] Add "limit 1" to the end of prepare sql of pushdown, to make it speed 
up

[cheng.wang] minor, fix UT

[liyang] minor, fix CacheServiceTest

[245915794] KYLIN-2761 refactor TableACL so that it can apply to role(user 
group in

[245915794] KYLIN-2980, Remove getKey/Value setKey/Value from Kylin's Pair.

[245915794] KYLIN-2960 sync LDAP user's group(Authorities) and move 
getAuthorities

[liyang] minor, CubeInstance add displayName, change cube display name in 
monitor

[liyang] #1961 complete batch api.

[liyang] supoort partition column type integer and date.

[cheng.wang] minor, refine CubeMigrationCLI

[github] #2732 take effect project config kylin.query.force-limit (#3205)

[245915794] minor, fix a typo that leads get user's permission error and add ut.

[245915794] minor, repair CubeMetaExtractor that can not retrieve 
tables/tableExds.

[ma] #1507 initial version

[event.dimlas] Minor, refine 

Build failed in Jenkins: Kylin-Master-JDK-1.7 #940

[Apache Jenkins Server]

See 


Changes:

[event.dimlas] Minor: do not hardcode DefaultQueryTransformer sequence.

[event.dimlas] Minor: Fix UT

[ma] Hide columns begin with "_KY_" for prepare queries

[liyang] minor, add project to cube list api

[github] minor, fix UT

[chen] kylin access for cube and model (#2782)

[github] KYLIN-2775 fix non-default database issue in sample model/cube (#2791)

[github] minor update KafkaSource log

[chen] remove access condition of modeler (#2798)

[zjsynce] Throw exception from pushdown instead of exception from calcite

[liyang] minor, append context info in Rel writer

[liyang] KYLIN-2930 make tablescan phy type always ARRAY

[chen] KYLIN-2756 Let 'LIMIT' be optional in 'inspect' page (#2813)

[lidong] minor, support real admin ldap-group name

[github] KYLIN-2945 global dict specific info change to use relative path 
(#2790)

[zjsynce] Shade commons-dbcp library in kylin-job jar

[liyang] fix old spark property in kylin.properties

[liyang] KYLIN-2944 HLLCSerializer, RawSerializer, PercentileSerializer returns

[liyang] KYLIN-2600 fix Incorrectly setting the range start

[liyang] minor, fix union query for ci

[rogershijicheng] KYLIN-2945 fix CI

[liyang] refactor, move HDFSResourceStore to the right module and others

[chen]  minor,kylin acl debug (#2874)

[zjsynce] Support function with name percentile_approx

[ma] minor, refine CubeMigrationCLI

[zjsynce] Shade commons-pool and commons-jocl library in kylin-job jar

[liyang] minor, refine TableExtDesc

[liyang] minor, export correct kylinConfig in ProjectInstance

[100650920] Revert "KYLIN-2777 Should throw error when sql contains non-equi 
left

[100650920] KYLIN-2777  Add test cases

[ma] minor, refine DiagnosisService

[550175214] minor, fix bug (#2908)

[event.dimlas] #2857 need pre processing in HivePushDownConverter

[liyang] Minor, set setCubeInstance func public.

[liyang] Return update date of cube response to the cube desc's update time

[liyang] Order models with data models' modified times

[liyang] minor refactor on PartitionDesc, make sure start time condition is

[liyang] bug fix in SegmentGTStartAndEnd

[zjsynce] Add function for doing pushdown query directly in KylinTestBase

[lidong] minor, update calcite dependency

[liyang] minor, fix log message in Broadcaster

[liyang] Build not a cube without date partition column.

[event.dimlas] Move out ComputedColumnDesc

[liyang] move NoRealizationFoundException RoutingIndicatorException

[liyang] KYLIN-2985 Cache temp json file created by each Calcite Connection

[liyang] KYLIN-2982 Avoid upgrade column in OLAPTable

[liyang] KYLIN-2982 update sum measure default return type

[github] minor, simplify OLAPSchemaFactory log

[shaofengshi] KYLIN-2991 fix NumberFormatException if partitionDateFormat is 
like

[lidong] minor, updates on OLAPContext

[shaofengshi] KYLIN-2797 Remove MR engine V1

[shaofengshi] KYLIN-2796 Remove the legacy "statisticsenabled" codes

[event.dimlas] #3062 pre-process MIN(1) and MAX(1)

[rogershijicheng] minor, make calcite rule configurable

[zjsynce] Do not do update query in push down engine for prepare query

[zjsynce] Add "limit 1" to the end of prepare sql of pushdown, to make it speed 
up

[liuyiming.vip] minor, drop invalid kylin.job.step.timeout in kylin.properties

[liuyiming.vip] KYLIN-1319, drop deprecated HadoopStatusChecker

[cheng.wang] Revert "Add "limit 1" to the end of prepare sql of pushdown, to 
make it

[cheng.wang] Revert "Do not do update query in push down engine for prepare 
query"

[ma] minor refactor

[liyang] KYLIN-2794 MultipleDictionaryValueEnumerator consider value order based

[rogershijicheng] KYLIN-3036 Allow complex column type when loading source table

[github] KYLIN-3038 cannot support sum of type-converted column SQL (#3124)

[liyang] minor, rename percentile to percentile_approx

[lidong] KYLIN-3040 fix segment name when refreshing non-partitioned cube

[zjsynce] Do not do update query in push down engine for prepare query

[zjsynce] Add "limit 1" to the end of prepare sql of pushdown, to make it speed 
up

[cheng.wang] minor, fix UT

[liyang] minor, fix CacheServiceTest

[245915794] KYLIN-2761 refactor TableACL so that it can apply to role(user 
group in

[245915794] KYLIN-2980, Remove getKey/Value setKey/Value from Kylin's Pair.

[245915794] KYLIN-2960 sync LDAP user's group(Authorities) and move 
getAuthorities

[liyang] minor, CubeInstance add displayName, change cube display name in 
monitor

[liyang] #1961 complete batch api.

[liyang] supoort partition column type integer and date.

[cheng.wang] minor, refine CubeMigrationCLI

[github] #2732 take effect project config kylin.query.force-limit (#3205)

[245915794] minor, fix a typo that leads get user's permission error and add ut.

[245915794] minor, repair CubeMetaExtractor that can not retrieve 
tables/tableExds.

[ma] #1507 initial version

[event.dimlas] Minor, refine 

[14/50] [abbrv] kylin git commit: Running task's status should be set to READY after metastore recover

[liyang]

Running task's status should be set to READY after metastore recover


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0942651
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0942651
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0942651

Branch: refs/heads/master
Commit: c09426515918b53495811ec0de6b3a21cb49f383
Parents: 05610de
Author: nichunen 
Authored: Wed Dec 13 16:14:29 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 13 17:23:48 2017 +0800

--
 .../kylin/job/execution/ExecutableManager.java  | 31 +---
 .../job/impl/threadpool/DefaultScheduler.java   |  2 +-
 2 files changed, 28 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0942651/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 9f67a2b..6110573 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -61,7 +61,7 @@ public class ExecutableManager {
 }
 
 // 

-
+
 private final KylinConfig config;
 private final ExecutableDao executableDao;
 
@@ -223,7 +223,8 @@ public class ExecutableManager {
  * @param expectedClass
  * @return
  */
-public List getAllAbstractExecutables(long 
timeStartInMillis, long timeEndInMillis, Class 
expectedClass) {
+public List getAllAbstractExecutables(long 
timeStartInMillis, long timeEndInMillis,
+Class expectedClass) {
 try {
 List ret = Lists.newArrayList();
 for (ExecutablePO po : executableDao.getJobs(timeStartInMillis, 
timeEndInMillis)) {
@@ -395,7 +396,8 @@ public class ExecutableManager {
 ExecutableState oldStatus = 
ExecutableState.valueOf(jobOutput.getStatus());
 if (newStatus != null && oldStatus != newStatus) {
 if (!ExecutableState.isValidStateTransfer(oldStatus, 
newStatus)) {
-throw new IllegalStateTranferException("there is no valid 
state transfer from:" + oldStatus + " to:" + newStatus + ", job id: " + jobId);
+throw new IllegalStateTranferException("there is no valid 
state transfer from:" + oldStatus + " to:"
++ newStatus + ", job id: " + jobId);
 }
 jobOutput.setStatus(newStatus.toString());
 }
@@ -413,6 +415,26 @@ public class ExecutableManager {
 }
 }
 
+public void forceKillJob(String jobId) {
+try {
+final ExecutableOutputPO jobOutput = 
executableDao.getJobOutput(jobId);
+jobOutput.setStatus(ExecutableState.ERROR.toString());
+List tasks = executableDao.getJob(jobId).getTasks();
+
+for (ExecutablePO task : tasks) {
+if 
(executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) {
+continue;
+} else if 
(executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) {
+updateJobOutput(task.getId(), ExecutableState.READY, 
Maps. newHashMap(), "");
+}
+break;
+}
+executableDao.updateJobOutput(jobOutput);
+} catch (PersistentException e) {
+throw new RuntimeException(e);
+}
+}
+
 //for migration only
 //TODO delete when migration finished
 public void resetJobOutput(String jobId, ExecutableState state, String 
output) {
@@ -495,7 +517,8 @@ public class ExecutableManager {
 }
 }
 
-private AbstractExecutable parseToAbstract(ExecutablePO executablePO, 
Class expectedClass) {
+private AbstractExecutable parseToAbstract(ExecutablePO executablePO,
+Class expectedClass) {
 if (executablePO == null) {
 logger.warn("executablePO is null");
 return null;

http://git-wip-us.apache.org/repos/asf/kylin/blob/c0942651/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index f5360da..ec5f552 100644
--- 

[40/50] [abbrv] kylin git commit: minor refactor on metadata

[liyang]

minor refactor on metadata


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8ddd820
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8ddd820
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8ddd820

Branch: refs/heads/master
Commit: b8ddd820383881fedd6ea617874887e0b0b32734
Parents: 826788a
Author: Hongbin Ma 
Authored: Thu Dec 28 16:15:49 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 28 17:37:18 2017 +0800

--
 .../org/apache/kylin/common/StorageURLTest.java |  18 ++-
 .../common/util/AbstractKylinTestCase.java  |   4 +
 .../apache/kylin/metadata/model/ColumnDesc.java |   5 +-
 .../kylin/metadata/model/DataModelDesc.java |  14 ++-
 .../kylin/metadata/model/DataModelManager.java  |   6 +-
 .../apache/kylin/metadata/model/JoinsTree.java  |  24 +++-
 .../metadata/model/tool/CalciteParser.java  |  37 ++
 .../kylin/metadata/model/JoinsTreeTest.java |   2 +-
 .../kylin/model/tool/CalciteParserTest.java | 122 ---
 .../apache/kylin/query/ITKylinQueryTest.java|   2 +-
 .../org/apache/kylin/query/KylinTestBase.java   |   4 +
 .../test/resources/query/sql_limit/query01.sql  |   2 +-
 .../test/resources/query/sql_limit/query03.sql  |   6 +-
 .../apache/kylin/rest/request/ModelRequest.java |   8 ++
 .../apache/kylin/rest/service/ModelService.java |   5 +-
 .../kylin/rest/util/Log4jConfigListener.java|   1 +
 16 files changed, 121 insertions(+), 139 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b8ddd820/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java 
b/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java
index eaa7548..643cc30 100644
--- a/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java
@@ -63,7 +63,23 @@ public class StorageURLTest {
 }
 
 @Test
+public void testHDFS() {
+{
+StorageURL id = new StorageURL(
+
"master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106");
+
+assertEquals("master_ci_instance", id.getIdentifier());
+assertEquals("hdfs", id.getScheme());
+assertEquals(1, id.getAllParameters().size());
+assertEquals(
+
"master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106",
+id.toString());
+}
+}
+
+@Test
 public void testEdgeCases() {
+
 {
 StorageURL id = new StorageURL("");
 assertEquals("kylin_metadata", id.getIdentifier());
@@ -87,7 +103,7 @@ public class StorageURLTest {
 assertEquals("hello@hbase,a", id.toString());
 }
 }
-
+
 @Test
 public void testValueOfCache() {
 StorageURL id1 = StorageURL.valueOf("hello@hbase");

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8ddd820/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 37790d2..7cbf5b6 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -26,6 +26,10 @@ import org.apache.kylin.common.KylinConfig;
  */
 public abstract class AbstractKylinTestCase {
 
+static {
+System.setProperty("needCheckCC", "true");
+}
+
 public abstract void createTestMetadata(String... overlayMetadataDirs) 
throws Exception;
 
 public abstract void cleanupTestMetadata() throws Exception;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8ddd820/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 03c89c3..b8de534 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -57,6 +57,10 @@ public class ColumnDesc implements Serializable {
   

[42/50] [abbrv] kylin git commit: KYLIN-3136, enhance protection when job's status is illegal

[liyang]

KYLIN-3136, enhance protection when job's status is illegal


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a6d1ab02
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a6d1ab02
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a6d1ab02

Branch: refs/heads/master
Commit: a6d1ab02234bcc021faed77fac17d74d3c1c7e30
Parents: 4eb6cd4
Author: Cheng Wang 
Authored: Thu Dec 28 18:52:52 2017 +0800
Committer: Li Yang 
Committed: Thu Dec 28 21:34:52 2017 -0600

--
 .../job/execution/DefaultChainedExecutable.java | 52 
 .../apache/kylin/job/RunningTestExecutable.java | 39 +++
 .../job/impl/threadpool/BaseSchedulerTest.java  |  3 --
 .../impl/threadpool/DefaultSchedulerTest.java   | 20 
 4 files changed, 102 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a6d1ab02/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index cbd49ae..8795e4c 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -43,7 +43,7 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 sub.initConfig(config);
 }
 }
-
+
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
 List executables = getTasks();
@@ -58,7 +58,8 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 // the job is paused
 break;
 } else if (state == ExecutableState.ERROR) {
-throw new IllegalStateException("invalid subtask state, 
subtask:" + subTask.getName() + ", state:" + subTask.getStatus());
+throw new IllegalStateException(
+"invalid subtask state, subtask:" + subTask.getName() 
+ ", state:" + subTask.getStatus());
 }
 if (subTask.isRunnable()) {
 return subTask.execute(context);
@@ -88,7 +89,7 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 @Override
 protected void onExecuteFinished(ExecuteResult result, ExecutableContext 
executableContext) {
 ExecutableManager mgr = getManager();
-
+
 if (isDiscarded()) {
 setEndTime(System.currentTimeMillis());
 notifyUserStatusChange(executableContext, 
ExecutableState.DISCARDED);
@@ -99,19 +100,26 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 List jobs = getTasks();
 boolean allSucceed = true;
 boolean hasError = false;
-boolean hasRunning = false;
 boolean hasDiscarded = false;
 for (Executable task : jobs) {
+if (task.getStatus() == ExecutableState.RUNNING) {
+logger.error(
+"There shouldn't be a running subtask[jobId: {}, 
jobName: {}], \n"
++ "it might cause endless state, will 
retry to fetch subtask's state.",
+task.getId(), task.getName());
+boolean retryRet = retryFetchTaskStatus(task);
+if (false == retryRet)
+hasError = true;
+}
+
 final ExecutableState status = task.getStatus();
+
 if (status == ExecutableState.ERROR) {
 hasError = true;
 }
 if (status != ExecutableState.SUCCEED) {
 allSucceed = false;
 }
-if (status == ExecutableState.RUNNING) {
-hasRunning = true;
-}
 if (status == ExecutableState.DISCARDED) {
 hasDiscarded = true;
 }
@@ -124,8 +132,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 setEndTime(System.currentTimeMillis());
 mgr.updateJobOutput(getId(), ExecutableState.ERROR, null, 
null);
 notifyUserStatusChange(executableContext, 
ExecutableState.ERROR);
-} else if (hasRunning) {
-mgr.updateJobOutput(getId(), ExecutableState.RUNNING, null, 
null);
 } else if (hasDiscarded) {
 

[28/50] [abbrv] kylin git commit: KYLIN-3128 Configs for allowing export query results for admin/nonadmin user

[liyang]

KYLIN-3128 Configs for allowing export query results for admin/nonadmin user


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bcda327f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bcda327f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bcda327f

Branch: refs/heads/master
Commit: bcda327f734d4833d3baff11709376c14da779bf
Parents: e787d7c
Author: nichunen 
Authored: Fri Dec 22 17:22:08 2017 +0800
Committer: Hongbin Ma 
Committed: Fri Dec 22 18:11:43 2017 +0800

--
 .../org/apache/kylin/common/KylinConfigBase.java  | 10 +-
 .../src/main/resources/kylin-defaults.properties  |  4 
 examples/test_case_data/sandbox/kylin.properties  |  3 +++
 .../kylin/rest/controller/QueryController.java| 12 
 .../java/org/apache/kylin/rest/msg/Message.java   |  4 
 webapp/app/js/controllers/query.js|  5 -
 webapp/app/js/services/admin.js   |  1 +
 webapp/app/js/services/kylinProperties.js | 18 +-
 webapp/app/partials/query/query_detail.html   |  2 +-
 9 files changed, 55 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bcda327f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 21362e9..5620ac0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1367,6 +1367,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
 }
 
+public boolean isAdminUserExportAllowed() {
+return 
Boolean.parseBoolean(getOptional("kylin.web.export-allow-admin", "true"));
+}
+
+public boolean isNoneAdminUserExportAllowed() {
+return 
Boolean.parseBoolean(getOptional("kylin.web.export-allow-other", "true"));
+}
+
 // 

 // RESTCLIENT
 // 

@@ -1412,6 +1420,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 
"kylin.web.timezone,kylin.query.cache-enabled,kylin.env,kylin.web.hive-limit,kylin.storage.default,kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic,"
 + 
"kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,kylin.web.help.3,"
 + 
"kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,kylin.security.profile,"
-+ "kylin.htrace.show-gui-trace-toggle");
++ 
"kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other");
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bcda327f/core-common/src/main/resources/kylin-defaults.properties
--
diff --git a/core-common/src/main/resources/kylin-defaults.properties 
b/core-common/src/main/resources/kylin-defaults.properties
index c7afc19..e19ff2e 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -45,6 +45,10 @@ kylin.web.timezone=GMT+8
 
 kylin.web.cross-domain-enabled=true
 
+#allow user to export query result
+kylin.web.export-allow-admin=true
+kylin.web.export-allow-other=true
+
 #max connections of one route
 kylin.restclient.connection.default-max-per-route=20
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/bcda327f/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 00fea26..a6d4adc 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -153,6 +153,9 @@ kylin.web.help.1=odbc|ODBC Driver|
 kylin.web.help.2=tableau|Tableau Guide|
 kylin.web.help.3=onboard|Cube Design Tutorial|
 
+#allow user to export query result
+kylin.web.export-allow-admin=true
+kylin.web.export-allow-other=true
 
 ### OTHER ###
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/bcda327f/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java

[03/50] [abbrv] kylin git commit: minor, refine relization cost calculation

[liyang]

minor, refine relization cost calculation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bace2d20
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bace2d20
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bace2d20

Branch: refs/heads/master
Commit: bace2d20b515ef0d458b2579d1923a7fa19d12a8
Parents: 221caaa
Author: Roger Shi 
Authored: Fri Dec 8 21:42:31 2017 +0800
Committer: Roger Shi 
Committed: Mon Dec 11 09:42:38 2017 +0800

--
 .../java/org/apache/kylin/cube/CubeInstance.java | 10 +-
 .../kylin/metadata/realization/IRealization.java |  2 ++
 .../kylin/storage/hybrid/HybridInstance.java |  9 +
 .../kylin/query/routing/RealizationChooser.java  | 19 +--
 .../rest/controller/CubeControllerTest.java  |  3 +--
 5 files changed, 22 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bace2d20/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 17d8a03..d1c5166 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -92,6 +92,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 private String descName;
 @JsonProperty("display_name")
 private String displayName;
+// DEPRECATED: the cost should be calculated in runtime
 // Mark cube priority for query
 @JsonProperty("cost")
 private int cost = 50;
@@ -248,8 +249,15 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 this.displayName = displayName;
 }
 
+@Override
 public int getCost() {
-return cost;
+int countedDimensionNum = getRowKeyColumnCount();
+int c = countedDimensionNum * COST_WEIGHT_DIMENSION + 
getMeasures().size() * COST_WEIGHT_MEASURE;
+for (JoinTableDesc join : getModel().getJoinTables()) {
+if (join.getJoin().isInnerJoin())
+c += CubeInstance.COST_WEIGHT_INNER_JOIN;
+}
+return c;
 }
 
 public void setCost(int cost) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/bace2d20/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index aafc0f0..cc9f1af 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -63,4 +63,6 @@ public interface IRealization extends IStorageAware {
 public boolean supportsLimitPushDown();
 
 public KylinConfig getConfig();
+
+public int getCost();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bace2d20/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index df68e10..14ef524 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -200,6 +200,15 @@ public class HybridInstance extends RootPersistentEntity 
implements IRealization
 }
 
 @Override
+public int getCost() {
+int c = Integer.MAX_VALUE;
+for (IRealization realization : getRealizations()) {
+c = Math.min(realization.getCost(), c);
+}
+return c;
+}
+
+@Override
 public RealizationType getType() {
 return RealizationType.HYBRID;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bace2d20/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java 
b/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index 89859b2..1493c84 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -26,18 

[19/50] [abbrv] kylin git commit: minor, refine generateAceResponses.

[liyang]

minor, refine  generateAceResponses.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f15da9e6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f15da9e6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f15da9e6

Branch: refs/heads/master
Commit: f15da9e6ca64232ecb6153396b4bb2656c36fbc8
Parents: 3a3d11a
Author: tttMelody <245915...@qq.com>
Authored: Tue Dec 12 20:42:20 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Thu Dec 14 19:07:28 2017 +0800

--
 .../kylin/rest/service/AccessService.java   | 44 +++-
 1 file changed, 43 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f15da9e6/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 3b3b1df..3732c15 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.AclEntity;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
@@ -314,19 +315,60 @@ public class AccessService {
 }
 }
 
-public List generateAceResponses(Acl acl) {
+public List generateAceResponsesByFuzzMatching(Acl 
acl, String nameSeg, boolean isCaseSensitive) {
 if (null == acl) {
 return Collections.emptyList();
 }
 
 List result = new 
ArrayList();
 for (AccessControlEntry ace : acl.getEntries()) {
+if (nameSeg != null && !needAdd(nameSeg, isCaseSensitive, 
getName(ace.getSid( {
+continue;
+}
 result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), 
ace.getPermission(), ace.isGranting()));
 }
 
 return result;
 }
 
+private boolean needAdd(String nameSeg, boolean isCaseSensitive, String 
name) {
+return isCaseSensitive && StringUtils.contains(name, nameSeg)
+|| !isCaseSensitive && StringUtils.containsIgnoreCase(name, 
nameSeg);
+}
+
+private static String getName(Sid sid) {
+if (sid instanceof PrincipalSid) {
+return ((PrincipalSid) sid).getPrincipal();
+} else {
+return ((GrantedAuthoritySid) sid).getGrantedAuthority();
+}
+}
+
+public List generateAceResponses(Acl acl) {
+return generateAceResponsesByFuzzMatching(acl, null, false);
+}
+
+public List getAllAclSids(Acl acl, String type) {
+if (null == acl) {
+return Collections.emptyList();
+}
+
+List result = new ArrayList<>();
+for (AccessControlEntry ace : acl.getEntries()) {
+String name = null;
+if (type.equalsIgnoreCase("user") && ace.getSid() instanceof 
PrincipalSid) {
+name = ((PrincipalSid) ace.getSid()).getPrincipal();
+}
+if (type.equalsIgnoreCase("group") && ace.getSid() instanceof 
GrantedAuthoritySid) {
+name = ((GrantedAuthoritySid) 
ace.getSid()).getGrantedAuthority();
+}
+if (!StringUtils.isBlank(name)) {
+result.add(name);
+}
+}
+return result;
+}
+
 /**
  * Protect admin permission granted to acl owner.
  *

[26/50] [abbrv] kylin git commit: KYLIN-3044, KYLIN-3052, support SQL Server & Redshift as kylin data source

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/e787d7cd/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
--
diff --git 
a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
 
b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
new file mode 100644
index 000..a5516ab
--- /dev/null
+++ 
b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadataTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.source.jdbc.metadata;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.kylin.source.hive.DBConnConf;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SQLServerJdbcMetadataTest extends DefaultJdbcMetadataTest {
+
+@Before
+public void setup() {
+dbConnConf = new DBConnConf();
+dbConnConf.setUrl("jdbc:sqlserver://fakehost:1433;database=testdb");
+dbConnConf.setDriver("com.microsoft.sqlserver.jdbc.SQLServerDriver");
+dbConnConf.setUser("user");
+dbConnConf.setPass("pass");
+jdbcMetadata = new SQLServerJdbcMetadata(dbConnConf);
+
+setupProperties();
+}
+
+@Test
+public void testListDatabases() throws SQLException {
+ResultSet rs = mock(ResultSet.class);
+when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false);
+
when(rs.getString("TABLE_SCHEM")).thenReturn("schema1").thenReturn("schema2");
+
when(rs.getString("TABLE_CATALOG")).thenReturn("catalog1").thenReturn("testdb");
+
+when(connection.getCatalog()).thenReturn("testdb");
+when(connection.getMetaData()).thenReturn(dbmd);
+when(dbmd.getSchemas("testdb", "%")).thenReturn(rs);
+
+List dbs = jdbcMetadata.listDatabases();
+
+Assert.assertEquals(1, dbs.size());
+Assert.assertEquals("schema2", dbs.get(0));
+}
+
+@Test(expected = IllegalArgumentException.class)
+public void testListDatabasesWithoutSpecificDB() throws SQLException {
+when(connection.getCatalog()).thenReturn("");
+jdbcMetadata.listDatabases();
+}
+}

[02/50] [abbrv] kylin git commit: minor, fix bug

[liyang]

minor, fix bug


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/221caaab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/221caaab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/221caaab

Branch: refs/heads/master
Commit: 221caaab80a910639c9df3a3d8934de558e76635
Parents: 3ebd15d
Author: Cheng Wang 
Authored: Sat Dec 9 23:43:33 2017 +0800
Committer: Ni Chunen 
Committed: Mon Dec 11 09:15:35 2017 +0800

--
 .../kylin/tool/AbstractInfoExtractor.java   | 28 ++--
 1 file changed, 20 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/221caaab/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
--
diff --git 
a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java 
b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
index 28f3c72..94df3bc 100644
--- a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
@@ -43,17 +43,20 @@ public abstract class AbstractInfoExtractor extends 
AbstractApplication {
 private static final Logger logger = 
LoggerFactory.getLogger(AbstractInfoExtractor.class);
 
 @SuppressWarnings("static-access")
-private static final Option OPTION_DEST = 
OptionBuilder.withArgName("destDir").hasArg().isRequired(true).withDescription("specify
 the dest dir to save the related information").create("destDir");
+private static final Option OPTION_DEST = 
OptionBuilder.withArgName("destDir").hasArg().isRequired(true)
+.withDescription("specify the dest dir to save the related 
information").create("destDir");
 
 @SuppressWarnings("static-access")
-private static final Option OPTION_COMPRESS = 
OptionBuilder.withArgName("compress").hasArg().isRequired(false).withDescription("specify
 whether to compress the output with zip. Default true.").create("compress");
+private static final Option OPTION_COMPRESS = 
OptionBuilder.withArgName("compress").hasArg().isRequired(false)
+.withDescription("specify whether to compress the output with zip. 
Default true.").create("compress");
 
 @SuppressWarnings("static-access")
-private static final Option OPTION_SUBMODULE = 
OptionBuilder.withArgName("submodule").hasArg().isRequired(false).withDescription("specify
 whether this is a submodule of other CLI tool").create("submodule");
+private static final Option OPTION_SUBMODULE = 
OptionBuilder.withArgName("submodule").hasArg().isRequired(false)
+.withDescription("specify whether this is a submodule of other CLI 
tool").create("submodule");
 
 @SuppressWarnings("static-access")
-private static final Option OPTION_PACKAGETYPE = 
OptionBuilder.withArgName("packagetype").hasArg().isRequired(false).withDescription("specify
 the package type").create("packagetype");
-
+private static final Option OPTION_PACKAGETYPE = 
OptionBuilder.withArgName("packagetype").hasArg().isRequired(false)
+.withDescription("specify the package type").create("packagetype");
 
 private static final String DEFAULT_PACKAGE_TYPE = "base";
 private static final String[] COMMIT_SHA1_FILES = { "commit_SHA1", 
"commit.sha1" };
@@ -80,9 +83,17 @@ public abstract class AbstractInfoExtractor extends 
AbstractApplication {
 @Override
 protected void execute(OptionsHelper optionsHelper) throws Exception {
 String exportDest = 
optionsHelper.getOptionValue(options.getOption("destDir"));
-boolean shouldCompress = optionsHelper.hasOption(OPTION_COMPRESS) ? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : true;
-boolean isSubmodule = optionsHelper.hasOption(OPTION_SUBMODULE) ? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_SUBMODULE)) : false;
+boolean shouldCompress = optionsHelper.hasOption(OPTION_COMPRESS)
+? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS))
+: true;
+boolean isSubmodule = optionsHelper.hasOption(OPTION_SUBMODULE)
+? 
Boolean.valueOf(optionsHelper.getOptionValue(OPTION_SUBMODULE))
+: false;
 packageType = optionsHelper.getOptionValue(OPTION_PACKAGETYPE);
+
+if (packageType == null)
+packageType = DEFAULT_PACKAGE_TYPE;
+
 if (StringUtils.isEmpty(exportDest)) {
 throw new RuntimeException("destDir is not set, exit directly 
without extracting");
 }
@@ -91,7 +102,8 @@ public abstract class AbstractInfoExtractor extends 
AbstractApplication {
 }
 
 // create new folder to contain the output
-

[12/50] [abbrv] kylin git commit: Fix job status not changed to error after HBase Region Server is shutdown

[liyang]

Fix job status not changed to error after HBase Region Server is shutdown


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e359901
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e359901
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e359901

Branch: refs/heads/master
Commit: 1e35990184815c3ed0f655542d8c9af118560dae
Parents: 4629d84
Author: nichunen 
Authored: Tue Dec 12 22:21:10 2017 +0800
Committer: Ni Chunen 
Committed: Tue Dec 12 22:43:24 2017 +0800

--
 .../job/impl/threadpool/DefaultScheduler.java   | 11 +-
 .../kylin/job/NoErrorStatusExecutable.java  | 37 
 .../job/impl/threadpool/BaseSchedulerTest.java  | 16 +
 .../impl/threadpool/DefaultSchedulerTest.java   | 14 
 4 files changed, 77 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e359901/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index b87a839..f5360da 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -59,6 +59,7 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 private static final Logger logger = 
LoggerFactory.getLogger(DefaultScheduler.class);
 private volatile boolean initialized = false;
 private volatile boolean hasStarted = false;
+volatile boolean fetchFailed = false;
 private JobEngineConfig jobEngineConfig;
 
 private static DefaultScheduler INSTANCE = null;
@@ -102,7 +103,12 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 } else if (output.getState() == 
ExecutableState.STOPPED) {
 nStopped++;
 } else {
-nOthers++;
+if (fetchFailed) {
+executableManager.updateJobOutput(id, 
ExecutableState.ERROR, null, null);
+nError++;
+} else {
+nOthers++;
+}
 }
 continue;
 }
@@ -122,10 +128,13 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 logger.warn(jobDesc + " fail to schedule", ex);
 }
 }
+
+fetchFailed = false;
 logger.info("Job Fetcher: " + nRunning + " should running, " + 
runningJobs.size() + " actual running, "
 + nStopped + " stopped, " + nReady + " ready, " + 
nSUCCEED + " already succeed, " + nError
 + " error, " + nDiscarded + " discarded, " + nOthers + 
" others");
 } catch (Exception e) {
+fetchFailed = true;
 logger.warn("Job Fetcher caught a exception ", e);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e359901/core-job/src/test/java/org/apache/kylin/job/NoErrorStatusExecutable.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/NoErrorStatusExecutable.java 
b/core-job/src/test/java/org/apache/kylin/job/NoErrorStatusExecutable.java
new file mode 100644
index 000..a203eb7
--- /dev/null
+++ b/core-job/src/test/java/org/apache/kylin/job/NoErrorStatusExecutable.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.job.execution.ExecutableContext;
+

[11/50] [abbrv] kylin git commit: 3403 Security enhancement for /api/admin/config (#3484)

[liyang]

3403 Security enhancement for /api/admin/config (#3484)

* KYLIN-1664 add whitelist for non-admin kylin properties

* 3403 KYLIN-1664 add another api for getting public configurations


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4629d848
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4629d848
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4629d848

Branch: refs/heads/master
Commit: 4629d8483914e29f4bb50cfcc6ba09b064187a38
Parents: af15062
Author: Shaofeng Shi 
Authored: Tue Dec 12 15:37:31 2017 +0800
Committer: Zhixiong Chen 
Committed: Tue Dec 12 15:37:31 2017 +0800

--
 .../org/apache/kylin/common/KylinConfig.java| 29 
 .../apache/kylin/common/KylinConfigBase.java| 22 ++-
 .../apache/kylin/query/ITKylinQueryTest.java|  1 +
 .../kylin/rest/controller/AdminController.java  | 27 +++---
 .../kylin/rest/controller/BasicController.java  | 19 +
 .../apache/kylin/rest/service/AdminService.java | 13 +++--
 server/src/main/resources/kylinSecurity.xml |  1 +
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  2 +-
 8 files changed, 94 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4629d848/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index c638ec6..c662a16 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -30,6 +30,7 @@ import java.lang.reflect.Method;
 import java.net.URL;
 import java.nio.ByteOrder;
 import java.nio.charset.Charset;
+import java.util.Collection;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
@@ -439,11 +440,9 @@ public class KylinConfig extends KylinConfigBase {
 return copy;
 }
 
-public String exportToString() throws IOException {
-Properties allProps = getAllProperties();
-OrderedProperties orderedProperties = 
KylinConfig.buildSiteOrderedProps();
-
-final StringBuilder sb = new StringBuilder();
+public String exportAllToString() throws IOException {
+final Properties allProps = getProperties(null);
+final OrderedProperties orderedProperties = 
KylinConfig.buildSiteOrderedProps();
 
 for (Map.Entry entry : allProps.entrySet()) {
 String key = entry.getKey().toString();
@@ -454,10 +453,30 @@ public class KylinConfig extends KylinConfigBase {
 orderedProperties.setProperty(key, value);
 }
 }
+
+final StringBuilder sb = new StringBuilder();
 for (Map.Entry entry : orderedProperties.entrySet()) {
 sb.append(entry.getKey() + "=" + entry.getValue()).append('\n');
 }
 return sb.toString();
+
+}
+
+public String exportToString(Collection propertyKeys) throws 
IOException {
+Properties filteredProps = getProperties(propertyKeys);
+OrderedProperties orderedProperties = 
KylinConfig.buildSiteOrderedProps();
+
+for (String key : propertyKeys) {
+if (!filteredProps.containsKey(key)) {
+filteredProps.put(key, orderedProperties.getProperty(key, ""));
+}
+}
+
+final StringBuilder sb = new StringBuilder();
+for (Map.Entry entry : filteredProps.entrySet()) {
+sb.append(entry.getKey() + "=" + entry.getValue()).append('\n');
+}
+return sb.toString();
 }
 
 public void exportToFile(File file) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4629d848/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 1302247..8572fa3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -21,6 +21,7 @@ package org.apache.kylin.common;
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -125,12 +126,23 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 protected Properties 

[37/50] [abbrv] kylin git commit: #3505 fix timestampadd gap betweeen spark and calcite

[liyang]

#3505 fix timestampadd gap betweeen spark and calcite


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d86c2a08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d86c2a08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d86c2a08

Branch: refs/heads/master
Commit: d86c2a087514fd9fc3c192ba0f9668855d9c7aa1
Parents: 6c31ca9
Author: Hongbin Ma 
Authored: Tue Dec 5 21:27:24 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 27 20:05:54 2017 +0800

--
 .../apache/kylin/query/ITKylinQueryTest.java| 20 ++
 .../org/apache/kylin/query/KylinTestBase.java   |  4 +--
 .../query/sql_h2_uncapable/query01.sql  | 22 ---
 .../query/sql_h2_uncapable/query02.sql  | 22 ---
 .../query/sql_h2_uncapable/query06.sql  | 22 ---
 .../query/sql_h2_uncapable/query07.sql  | 22 ---
 .../query/sql_h2_uncapable/query08.sql  | 22 ---
 .../query/sql_h2_uncapable/query09.sql  | 22 ---
 .../resources/query/sql_timestamp/query01.sql   | 29 
 .../resources/query/sql_timestamp/query02.sql   | 28 +++
 .../resources/query/sql_timestamp/query06.sql   | 28 +++
 .../resources/query/sql_timestamp/query07.sql   | 28 +++
 .../resources/query/sql_timestamp/query08.sql   | 28 +++
 .../resources/query/sql_timestamp/query09.sql   | 28 +++
 .../resources/query/sql_timestamp/query11.sql   | 21 ++
 .../resources/query/sql_timestamp/query12.sql   | 21 ++
 .../resources/query/sql_timestamp/query13.sql   | 21 ++
 .../resources/query/sql_timestamp/query14.sql   | 21 ++
 pom.xml |  4 +--
 19 files changed, 271 insertions(+), 142 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d86c2a08/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index c468a9f..6dfdeb5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -83,7 +83,8 @@ public class ITKylinQueryTest extends KylinTestBase {
 try {
 
 Map toggles = Maps.newHashMap();
-toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR, 
StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 
10ms for every scan
+toggles.put(BackdoorToggles.DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR,
+
StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY.toString());//delay 
10ms for every scan
 BackdoorToggles.setToggles(toggles);
 
 
KylinConfig.getInstanceFromEnv().setProperty("kylin.storage.hbase.coprocessor-timeout-seconds",
 "3");
@@ -105,7 +106,8 @@ public class ITKylinQueryTest extends KylinTestBase {
 }
 
 protected void runTimeoutQueries() throws Exception {
-List sqlFiles = getFilesFromFolder(new 
File(getQueryFolderPrefix() + "src/test/resources/query/sql_timeout"), ".sql");
+List sqlFiles = getFilesFromFolder(
+new File(getQueryFolderPrefix() + 
"src/test/resources/query/sql_timeout"), ".sql");
 for (File sqlFile : sqlFiles) {
 try {
 runSQL(sqlFile, false, false);
@@ -205,14 +207,14 @@ public class ITKylinQueryTest extends KylinTestBase {
 Map toggles = Maps.newHashMap();
 toggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
 BackdoorToggles.setToggles(toggles);
-
+
 verifyResultRowColCount(getQueryFolderPrefix() + 
"src/test/resources/query/sql_verifyCount");
-
+
 } finally {
 BackdoorToggles.cleanToggles();
 }
 }
-
+
 @Test
 public void testVerifyContentQuery() throws Exception {
 verifyResultContent(getQueryFolderPrefix() + 
"src/test/resources/query/sql_verifyContent");
@@ -243,6 +245,11 @@ public class ITKylinQueryTest extends KylinTestBase {
 }
 
 @Test
+public void testTimeStampAdd() throws Exception {
+execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_timestamp", null, true);
+}
+
+@Test
 public void testCachedQuery() throws Exception {
 execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_cache", null, true);
 }
@@ -355,7 +362,8 @@ public class 

[31/50] [abbrv] kylin git commit: KYLIN-1664, add default properties and ut.

[liyang]

KYLIN-1664, add default properties and ut.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/19c4eb79
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/19c4eb79
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/19c4eb79

Branch: refs/heads/master
Commit: 19c4eb79576c102cb1475d7a96ff13db14496712
Parents: 17ce7d7
Author: tttMelody <245915...@qq.com>
Authored: Mon Dec 25 23:01:05 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Tue Dec 26 19:44:40 2017 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  1 +
 .../main/resources/kylin-defaults.properties| 16 -
 .../kylin/rest/controller/AdminController.java  | 15 +---
 .../apache/kylin/rest/service/AdminService.java | 15 
 .../kylin/rest/service/AdminServiceTest.java| 72 
 server/src/test/resources/kylin.properties  | 17 +
 6 files changed, 121 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/19c4eb79/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index c662a16..8872f45 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -55,6 +55,7 @@ public class KylinConfig extends KylinConfigBase {
  * Kylin properties file name
  */
 public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties";
+public static final String KYLIN_DEFAULT_CONF_PROPERTIES_FILE = 
"kylin-defaults.properties";
 public static final String KYLIN_CONF = "KYLIN_CONF";
 
 // static cached instances

http://git-wip-us.apache.org/repos/asf/kylin/blob/19c4eb79/core-common/src/main/resources/kylin-defaults.properties
--
diff --git a/core-common/src/main/resources/kylin-defaults.properties 
b/core-common/src/main/resources/kylin-defaults.properties
index d8b2d6a..51e5a2d 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -55,6 +55,21 @@ kylin.restclient.connection.default-max-per-route=20
 #max connections of one rest-client
 kylin.restclient.connection.max-total=200
 
+### PUBLIC CONFIG ###
+kylin.engine.default=2
+kylin.storage.default=2
+kylin.web.hive-limit=20
+kylin.web.help.length=4
+kylin.web.help.0=start|Getting Started|
+kylin.web.help.1=odbc|ODBC Driver|
+kylin.web.help.2=tableau|Tableau Guide|
+kylin.web.help.3=onboard|Cube Design Tutorial|
+kylin.web.link-streaming-guide=http://kylin.apache.org/
+kylin.htrace.show-gui-trace-toggle=false
+kylin.web.link-hadoop=
+kylin.web.link-diagnostic=
+kylin.web.contact-mail=
+kylin.server.external-acl-provider=
 
 ### SOURCE ###
 
@@ -244,7 +259,6 @@ kylin.security.saml.context-server-name=hostname
 kylin.security.saml.context-server-port=443
 kylin.security.saml.context-path=/kylin
 
-
 ### SPARK ENGINE CONFIGS ###
 
 # Hadoop conf folder, will export this as "HADOOP_CONF_DIR" to run spark-submit

http://git-wip-us.apache.org/repos/asf/kylin/blob/19c4eb79/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java
index bb71190..963a945 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java
@@ -19,12 +19,8 @@
 package org.apache.kylin.rest.controller;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
 
-import com.google.common.collect.Lists;
 import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.rest.msg.Message;
 import org.apache.kylin.rest.msg.MsgPicker;
@@ -87,21 +83,12 @@ public class AdminController extends BasicController {
 @RequestMapping(value = "/public_config", method = { RequestMethod.GET }, 
produces = { "application/json" })
 @ResponseBody
 public GeneralResponse getPublicConfig() throws IOException {
-final String whiteListProperties = 
KylinConfig.getInstanceFromEnv().getPropertiesWhiteList();
-
-Collection propertyKeys = Lists.newArrayList();
-if (StringUtils.isNotEmpty(whiteListProperties)) {
-

[06/50] [abbrv] kylin git commit: KYLIN-3092 Synchronize w/r operations on entity-caching managers

[liyang]

KYLIN-3092 Synchronize w/r operations on entity-caching managers


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f2f487fe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f2f487fe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f2f487fe

Branch: refs/heads/master
Commit: f2f487fe2dc868119562303cb8b9a0b630f704cf
Parents: b8d7987
Author: Li Yang 
Authored: Sat Dec 9 00:23:51 2017 +0800
Committer: Dong Li 
Committed: Mon Dec 11 10:16:35 2017 +0800

--
 .../org/apache/kylin/cube/CubeDescManager.java  |  311 +++--
 .../org/apache/kylin/cube/CubeInstance.java |   31 +-
 .../java/org/apache/kylin/cube/CubeManager.java | 1079 +-
 .../kylin/cube/cli/CubeSignatureRefresher.java  |2 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   29 +-
 .../kylin/metadata/TableMetadataManager.java|  583 +-
 .../kylin/metadata/TempStatementManager.java|  157 ++-
 .../org/apache/kylin/metadata/acl/TableACL.java |   11 +
 .../kylin/metadata/acl/TableACLManager.java |  111 +-
 .../metadata/cachesync/CachedCrudAssist.java|   72 +-
 .../kylin/metadata/model/DataModelDesc.java |7 +-
 .../kylin/metadata/model/DataModelManager.java  |  269 +++--
 .../metadata/model/ExternalFilterDesc.java  |5 +
 .../apache/kylin/metadata/model/TableDesc.java  |   79 +-
 .../kylin/metadata/model/TableExtDesc.java  |   38 +-
 .../kylin/metadata/project/ProjectInstance.java |2 +-
 .../kylin/metadata/project/ProjectManager.java  |   12 +-
 .../metadata/streaming/StreamingConfig.java |6 +
 .../metadata/streaming/StreamingManager.java|  183 +--
 .../metadata/TempStatementManagerTest.java  |6 +-
 .../streaming/StreamingManagerTest.java |   68 ++
 .../kylin/storage/hybrid/HybridInstance.java|   37 +-
 .../kylin/storage/hybrid/HybridManager.java |  133 ++-
 .../apache/kylin/engine/spark/SparkCubing.java  |4 +-
 .../DEFAULT_SESSION/temp_table1.json|1 +
 .../DEFAULT_SESSION/temp_table2.json|1 +
 .../kylin/provision/BuildCubeWithStream.java|2 +-
 .../kylin/rest/controller/ModelController.java  |2 +-
 .../kylin/rest/service/KafkaConfigService.java  |2 +-
 .../kylin/rest/service/StreamingService.java|4 +-
 .../kylin/source/kafka/KafkaConfigManager.java  |  166 +--
 .../kylin/source/kafka/config/KafkaConfig.java  |8 +-
 32 files changed, 1746 insertions(+), 1675 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index f724549..a58ba40 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -25,9 +25,9 @@ import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.AutoReadWriteLock;
+import org.apache.kylin.common.util.AutoReadWriteLock.AutoLock;
 import org.apache.kylin.cube.cuboid.CuboidManager;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
@@ -36,9 +36,9 @@ import org.apache.kylin.dimension.DictionaryDimEnc;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
 import org.apache.kylin.measure.topn.TopNMeasureType;
-import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.model.MeasureDesc;
@@ -59,8 +59,6 @@ public class CubeDescManager {
 
 private static final Logger logger = 
LoggerFactory.getLogger(CubeDescManager.class);
 
-public static final Serializer CUBE_DESC_SERIALIZER = new 
JsonSerializer(CubeDesc.class);
-
 public static CubeDescManager getInstance(KylinConfig config) {
 return config.getManager(CubeDescManager.class);
 }
@@ -69,20 +67,42 @@ public class CubeDescManager {
 static CubeDescManager newInstance(KylinConfig config) throws IOException {
 return new 

[36/50] [abbrv] kylin git commit: KYLIN-2982 minor refine

[liyang]

KYLIN-2982 minor refine


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3b200a78
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3b200a78
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3b200a78

Branch: refs/heads/master
Commit: 3b200a78fc8149376ae1e70404b8afc8bb67ce91
Parents: d86c2a0
Author: Hongbin Ma 
Authored: Thu Dec 14 14:52:37 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 27 20:05:54 2017 +0800

--
 .../apache/kylin/metadata/model/ColumnDesc.java | 11 +
 .../kylin/metadata/model/FunctionDesc.java  |  7 ++
 .../org/apache/kylin/metadata/tuple/Tuple.java  | 25 +++-
 3 files changed, 17 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3b200a78/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index b4381ea..03c89c3 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -59,7 +59,6 @@ public class ColumnDesc implements Serializable {
 
 // parsed from data type
 private DataType type;
-private DataType upgradedType;
 
 private TableDesc table;
 private int zeroBasedIndex = -1;
@@ -116,16 +115,8 @@ public class ColumnDesc implements Serializable {
 type = DataType.getType(datatype);
 }
 
-public void setUpgradedType(DataType upgradedType) {
-this.upgradedType = upgradedType;
-}
-
 public DataType getUpgradedType() {
-if (this.upgradedType == null) {
-return this.type;
-} else {
-return this.upgradedType;
-}
+return this.type;
 }
 
 public String getId() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3b200a78/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index c96713d..fa97596 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -154,15 +154,12 @@ public class FunctionDesc implements Serializable {
 }
 
 public DataType getRewriteFieldType() {
+
 if (getMeasureType() instanceof BasicMeasureType) {
 if (isMax() || isMin()) {
 return parameter.getColRefs().get(0).getType();
 } else if (isSum()) {
-if 
(parameter.getColRefs().get(0).getType().getName().equals(returnDataType.getName()))
 {
-return returnDataType;
-} else {
-return parameter.getColRefs().get(0).getType();
-}
+return parameter.getColRefs().get(0).getType();
 } else if (isCount()) {
 return DataType.getType("bigint");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3b200a78/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index b50a7e4..a299216 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -115,25 +115,27 @@ public class Tuple implements ITuple {
 // BigDecimal during cube build for best precision
 if ("double".equals(dataType) && fieldValue instanceof BigDecimal) {
 fieldValue = ((BigDecimal) fieldValue).doubleValue();
-} else if ("decimal".equals(dataType) && fieldValue instanceof 
BigDecimal) {
-fieldValue = normalizeDecimal((BigDecimal) fieldValue);
+} else if ("decimal".equals(dataType)) {
+if (fieldValue instanceof BigDecimal) {
+fieldValue = normalizeDecimal((BigDecimal) fieldValue);
+} else if (fieldValue instanceof Number) {
+fieldValue = new BigDecimal(((Number) 
fieldValue).doubleValue());
+}
+} else if ("float".equals(dataType) && fieldValue instanceof 
BigDecimal) {
+fieldValue = 

[24/50] [abbrv] kylin git commit: KYLIN-3092 fix cyclic lock

[liyang]

KYLIN-3092 fix cyclic lock


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f37fbce1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f37fbce1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f37fbce1

Branch: refs/heads/master
Commit: f37fbce17f6e25775877290ba1f0de076c62f32f
Parents: 9f603d3
Author: Li Yang 
Authored: Wed Dec 20 21:50:54 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 21 11:06:34 2017 +0800

--
 .../java/org/apache/kylin/common/util/HiveCmdBuilder.java |  4 ++--
 .../main/java/org/apache/kylin/cube/CubeDescManager.java  |  9 -
 .../kylin/job/impl/threadpool/DefaultScheduler.java   |  5 +++--
 .../org/apache/kylin/metadata/TableMetadataManager.java   | 10 +++---
 4 files changed, 12 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 10d6798..77dc579 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -188,8 +188,8 @@ public class HiveCmdBuilder {
 hiveConfFile = new File(path + File.separator + "conf", 
hiveConfFileName);
 }
 
-if (hiveConfFile == null || !hiveConfFile.exists()) {
-throw new RuntimeException("Failed to read " + HIVE_CONF_FILENAME 
+ ".xml");
+if (!hiveConfFile.exists()) {
+throw new RuntimeException("Missing config file: " + 
hiveConfFile.getAbsolutePath());
 }
 
 String fileUrl = 
OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath());

http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index a58ba40..a416499 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -46,7 +46,6 @@ import org.apache.kylin.metadata.model.ParameterDesc;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -166,14 +165,6 @@ public class CubeDescManager {
 if (ndesc.isBroken())
 throw new IllegalStateException("CubeDesc " + name + " is 
broken");
 
-// if related cube is in DESCBROKEN state before, change it back 
to DISABLED
-CubeManager cubeManager = CubeManager.getInstance(config);
-for (CubeInstance cube : cubeManager.getCubesByDesc(name)) {
-if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
-cube.init(config);
-}
-}
-
 return ndesc;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 78f10bf..c8ab7ea 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -163,8 +163,6 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 try (SetThreadName ignored = new SetThreadName("Scheduler %s Job 
%s",
 System.identityHashCode(DefaultScheduler.this), 
executable.getId())) {
 executable.execute(context);
-// trigger the next step asap
-fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
 } catch (ExecuteException e) {
 logger.error("ExecuteException job:" + executable.getId(), e);
 } catch (Exception e) {
@@ -172,6 +170,9 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 } finally {
 

[47/50] [abbrv] kylin git commit: Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
--
diff --cc 
engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 89bb11e,575c7b3..569b810
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@@ -54,20 -56,20 +56,17 @@@ public class FactDistinctColumnsMapper<
  BYTES
  }
  
- protected CuboidScheduler cuboidScheduler = null;
 -protected boolean collectStatistics = false;
  protected int nRowKey;
  private Integer[][] allCuboidsBitSet = null;
  private HLLCounter[] allCuboidsHLL = null;
  private Long[] cuboidIds;
- private HashFunction hf = null;
  private int rowCount = 0;
  private int samplingPercentage;
- //private ByteArray[] row_hashcodes = null;
- private long[] rowHashCodesLong = null;
  private ByteBuffer tmpbuf;
+ 
+ private CuboidStatCalculator[] cuboidStatCalculators;
+ 
  private static final Text EMPTY_TEXT = new Text();
--public static final byte MARK_FOR_PARTITION_COL = (byte) 0xFE;
--public static final byte MARK_FOR_HLL = (byte) 0xFF;
  
  private int partitionColumnIndex = -1;
  private boolean needFetchPartitionCol = true;
@@@ -81,70 -80,95 +77,93 @@@
  protected void doSetup(Context context) throws IOException {
  super.doSetup(context);
  tmpbuf = ByteBuffer.allocate(4096);
 -collectStatistics = 
Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
 -if (collectStatistics) {
 -samplingPercentage = Integer
 -
.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
 -nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
 -
 -Set cuboidIdSet = 
Sets.newHashSet(cubeSeg.getCuboidScheduler().getAllCuboidIds());
 -if (StatisticsDecisionUtil.isAbleToOptimizeCubingPlan(cubeSeg)) {
 -// For cube planner, for every prebuilt cuboid, its related 
row count stats should be calculated
 -// If the precondition for trigger cube planner phase one is 
satisfied, we need to calculate row count stats for mandatory cuboids.
 -
cuboidIdSet.addAll(cubeSeg.getCubeDesc().getMandatoryCuboids());
 -}
 -cuboidIds = cuboidIdSet.toArray(new Long[cuboidIdSet.size()]);
 -allCuboidsBitSet = CuboidUtil.getCuboidBitSet(cuboidIds, nRowKey);
+ 
 -allCuboidsHLL = new HLLCounter[cuboidIds.length];
 -for (int i = 0; i < cuboidIds.length; i++) {
 -allCuboidsHLL[i] = new 
HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision(), RegisterType.DENSE);
 -}
 +samplingPercentage = Integer
 +
.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
- cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
 +nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
  
- List cuboidIdList = Lists.newArrayList();
- List allCuboidsBitSetList = Lists.newArrayList();
- addCuboidBitSet(baseCuboidId, allCuboidsBitSetList, cuboidIdList);
- 
- allCuboidsBitSet = allCuboidsBitSetList.toArray(new 
Integer[cuboidIdList.size()][]);
- cuboidIds = cuboidIdList.toArray(new Long[cuboidIdList.size()]);
 -TblColRef partitionColRef = 
cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
 -if (partitionColRef != null) {
 -partitionColumnIndex = 
intermediateTableDesc.getColumnIndex(partitionColRef);
 -}
++Set cuboidIdSet = 
Sets.newHashSet(cubeSeg.getCuboidScheduler().getAllCuboidIds());
++if (StatisticsDecisionUtil.isAbleToOptimizeCubingPlan(cubeSeg)) {
++// For cube planner, for every prebuilt cuboid, its related row 
count stats should be calculated
++// If the precondition for trigger cube planner phase one is 
satisfied, we need to calculate row count stats for mandatory cuboids.
++cuboidIdSet.addAll(cubeSeg.getCubeDesc().getMandatoryCuboids());
++}
++cuboidIds = cuboidIdSet.toArray(new Long[cuboidIdSet.size()]);
++allCuboidsBitSet = CuboidUtil.getCuboidBitSet(cuboidIds, nRowKey);
  
 -// check whether need fetch the partition col values
 -if (partitionColumnIndex < 0) {
 -// if partition col not on cube, no need
 -needFetchPartitionCol = false;
 -} else {
 -needFetchPartitionCol = true;
 -}
 -//for KYLIN-2518 backward compatibility
 -

[25/50] [abbrv] kylin git commit: Add null pointer check for ProjectManager#getProject to avoid NPE

[liyang]

Add null pointer check for ProjectManager#getProject to avoid NPE


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca42b92b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca42b92b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca42b92b

Branch: refs/heads/master
Commit: ca42b92b6d86eb6cb3eb22a212e77f2155b19890
Parents: f37fbce
Author: nichunen 
Authored: Thu Dec 21 11:42:43 2017 +0800
Committer: Li Yang 
Committed: Wed Dec 20 21:45:01 2017 -0600

--
 .../java/org/apache/kylin/metadata/project/ProjectManager.java   | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca42b92b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 1c0254e..0dd364d 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -132,6 +132,10 @@ public class ProjectManager {
 }
 
 public ProjectInstance getProject(String projectName) {
+// Null check is needed for ConcurrentMap does not supporting 
.get(null)
+if (projectName == null)
+return null;
+
 try (AutoLock lock = prjMapLock.lockForRead()) {
 return projectMap.get(projectName);
 }

[46/50] [abbrv] kylin git commit: Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
--
diff --cc 
server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
index 000,35ba615..ee9fdcd
mode 00,100644..100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
@@@ -1,0 -1,129 +1,129 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  * 
+  * http://www.apache.org/licenses/LICENSE-2.0
+  * 
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+ */
+ 
+ 
+ package org.apache.kylin.rest.controller;
+ 
++import java.util.List;
++
+ import org.apache.kylin.cube.CubeInstance;
+ import org.apache.kylin.metadata.project.ProjectInstance;
+ import org.apache.kylin.metrics.MetricsManager;
+ import org.apache.kylin.rest.request.SQLRequest;
+ import org.apache.kylin.rest.response.MetricsResponse;
+ import org.apache.kylin.rest.response.SQLResponse;
+ import org.apache.kylin.rest.service.CubeService;
+ import org.apache.kylin.rest.service.DashboardService;
+ import org.apache.kylin.rest.service.QueryService;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.beans.factory.annotation.Autowired;
+ import org.springframework.security.access.AccessDeniedException;
+ import org.springframework.stereotype.Controller;
+ import org.springframework.web.bind.annotation.PathVariable;
+ import org.springframework.web.bind.annotation.RequestMapping;
+ import org.springframework.web.bind.annotation.RequestMethod;
+ import org.springframework.web.bind.annotation.RequestParam;
+ import org.springframework.web.bind.annotation.ResponseBody;
+ 
 -import java.util.List;
 -
+ @Controller
+ @RequestMapping(value = "/dashboard")
+ public class DashboardController extends BasicController {
+ private static final Logger logger = 
LoggerFactory.getLogger(DashboardController.class);
+ 
+ @Autowired
+ private DashboardService dashboardService;
+ 
+ @Autowired
+ private QueryService queryService;
+ 
+ @Autowired
+ private CubeService cubeService;
+ 
+ @RequestMapping(value = "/metric/cube", method = { RequestMethod.GET })
+ @ResponseBody
+ public MetricsResponse getCubeMetrics(@RequestParam(value = 
"projectName", required = false) String projectName, @RequestParam(value = 
"cubeName", required = false) String cubeName) {
+ checkAuthorization(projectName);
+ return dashboardService.getCubeMetrics(projectName, cubeName);
+ }
+ 
+ @RequestMapping(value = "/metric/query", method = RequestMethod.GET)
+ @ResponseBody
+ public MetricsResponse getQueryMetrics(@RequestParam(value = 
"projectName", required = false) String projectName, @RequestParam(value = 
"cubeName", required = false) String cubeName, @RequestParam(value = 
"startTime") String startTime, @RequestParam(value = "endTime") String endTime) 
{
+ checkAuthorization(projectName);
+ MetricsResponse queryMetrics = new MetricsResponse();
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String sql = dashboardService.getQueryMetricsSQL(startTime, endTime, 
projectName, cubeName);
+ sqlRequest.setSql(sql);
 -SQLResponse sqlResponse = queryService.queryWithoutSecure(sqlRequest);
++SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
+ if(!sqlResponse.getIsException()){
+ queryMetrics.increase("queryCount", 
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
+ queryMetrics.increase("avgQueryLatency", 
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+ queryMetrics.increase("maxQueryLatency", 
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+ queryMetrics.increase("minQueryLatency", 
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+ }
+ return queryMetrics;
+ }
+ 
+ @RequestMapping(value = "/metric/job", method = RequestMethod.GET)
+ 

[50/50] [abbrv] kylin git commit: Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'

[liyang]

Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d50b269
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d50b269
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d50b269

Branch: refs/heads/master
Commit: 4d50b26972bb7bbaff852172990e0f189f987673
Parents: 7889cec 5f2eff6
Author: Li Yang 
Authored: Sat Dec 30 21:57:51 2017 +0800
Committer: Li Yang 
Committed: Sun Dec 31 19:57:05 2017 +0800

--
 README.md   |   2 +-
 build/bin/load-hive-conf.sh |   4 +-
 build/script/download-spark.sh  |  18 +-
 build/script/download-tomcat.sh |   4 +-
 .../apache/kylin/common/KylinConfigBase.java| 149 -
 .../org/apache/kylin/common/QueryContext.java   | 459 -
 .../org/apache/kylin/common/util/BytesUtil.java |   2 +-
 .../apache/kylin/common/util/HadoopUtil.java|  19 +
 .../org/apache/kylin/common/util/JsonUtil.java  |  11 +-
 .../java/org/apache/kylin/common/util/Pair.java |  16 +
 .../kylin-backward-compatibility.properties |   1 +
 .../main/resources/kylin-defaults.properties|  13 +
 core-cube/pom.xml   |   4 +
 .../org/apache/kylin/cube/CubeDescManager.java  |   4 +
 .../org/apache/kylin/cube/CubeInstance.java |  90 ++-
 .../java/org/apache/kylin/cube/CubeManager.java | 158 +++--
 .../java/org/apache/kylin/cube/CubeUpdate.java  |  14 +-
 .../kylin/cube/common/RowKeySplitter.java   |   2 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java|  10 +-
 .../apache/kylin/cube/cuboid/CuboidUtil.java|  85 +++
 .../kylin/cube/cuboid/TreeCuboidScheduler.java  |   3 +-
 .../cuboid/algorithm/CuboidRecommender.java |   6 +
 .../inmemcubing/AbstractInMemCubeBuilder.java   |  13 +-
 .../ConsumeBlockingQueueController.java |  84 +++
 .../cube/inmemcubing/DoggedCubeBuilder.java | 128 +---
 .../cube/inmemcubing/InMemCubeBuilder.java  | 102 +--
 .../InMemCubeBuilderInputConverter.java | 149 -
 .../kylin/cube/inmemcubing/InputConverter.java  |  69 ++
 .../cube/inmemcubing/InputConverterUnit.java|  42 ++
 .../InputConverterUnitForBaseCuboid.java|  64 ++
 .../InputConverterUnitForRawData.java   | 169 +
 .../RecordConsumeBlockingQueueController.java   |  80 +++
 .../org/apache/kylin/cube/kv/RowKeyDecoder.java |   4 +-
 .../kylin/cube/model/AggregationGroup.java  |  10 +-
 .../kylin/cube/model/CubeBuildTypeEnum.java |   7 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   | 111 +++-
 .../validation/rule/AggregationGroupRule.java   |   2 +-
 .../kylin/gridtable/GTAggregateScanner.java |   8 +-
 .../apache/kylin/gridtable/GTFilterScanner.java |   6 +
 .../org/apache/kylin/gridtable/GTRecord.java|   5 +
 .../kylin/cube/AggregationGroupRuleTest.java|   6 +-
 .../org/apache/kylin/cube/CubeDescTest.java |  31 +-
 .../org/apache/kylin/cube/CubeInstanceTest.java |  14 +-
 .../org/apache/kylin/cube/CubeManagerTest.java  | 109 +++
 .../kylin/cube/cuboid/CuboidUtilTest.java   |  57 ++
 .../ConsumeBlockingQueueControllerTest.java | 124 
 .../apache/kylin/cube/kv/RowKeyDecoderTest.java |   2 +-
 .../apache/kylin/cube/kv/RowKeyEncoderTest.java |   6 +-
 .../apache/kylin/dict/DictionaryGenerator.java  |  44 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |   4 +
 .../apache/kylin/dict/DictionaryManager.java|   4 +
 .../kylin/dict/GlobalDictionaryBuilder.java |  16 +-
 .../apache/kylin/dict/IDictionaryBuilder.java   |   2 +-
 .../dict/MultipleDictionaryValueEnumerator.java |  10 +-
 .../kylin/dict/Number2BytesConverter.java   |  14 +-
 .../global/SegmentAppendTrieDictBuilder.java|   9 +-
 .../kylin/dict/DictionaryProviderTest.java  |   2 +-
 .../apache/kylin/dict/TrieDictionaryTest.java   |  30 +-
 core-job/pom.xml|   4 +
 .../org/apache/kylin/engine/EngineFactory.java  |   4 +
 .../apache/kylin/engine/IBatchCubingEngine.java |   3 +
 .../org/apache/kylin/job/JoinedFlatTable.java   |  18 +-
 .../kylin/job/common/ShellExecutable.java   |   7 +-
 .../kylin/job/constant/ExecutableConstants.java |   6 +
 .../org/apache/kylin/job/dao/ExecutableDao.java |  18 +-
 .../org/apache/kylin/job/dao/ExecutablePO.java  |  11 +
 .../kylin/job/engine/JobEngineConfig.java   |  14 +
 .../kylin/job/exception/ShellException.java |  43 ++
 .../kylin/job/execution/AbstractExecutable.java |  50 +-
 .../job/execution/CheckpointExecutable.java | 122 
 .../job/execution/DefaultChainedExecutable.java |  30 +-
 .../kylin/job/execution/ExecutableManager.java  |  46 +-
 .../kylin/job/execution/ExecuteResult.java  |  31 +
 .../job/impl/threadpool/DefaultScheduler.java   | 170 -
 

[16/50] [abbrv] kylin git commit: KYLIN-3085 Makes sure no update on cached and shared CubeInstance

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e6b8c5/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
index e3a7586..37bf8ff 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
@@ -50,12 +50,12 @@ public class MergeOffsetStep extends AbstractExecutable {
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
 final CubeManager cubeManager = 
CubeManager.getInstance(context.getConfig());
-final CubeInstance cube = 
cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+final CubeInstance cubeCopy = 
cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite();
 final String segmentId = 
CubingExecutableUtil.getSegmentId(this.getParams());
-final CubeSegment segment = cube.getSegmentById(segmentId);
+final CubeSegment segCopy = cubeCopy.getSegmentById(segmentId);
 
-Preconditions.checkNotNull(segment, "Cube segment '" + segmentId + "' 
not found.");
-Segments mergingSegs = cube.getMergingSegments(segment);
+Preconditions.checkNotNull(segCopy, "Cube segment '" + segmentId + "' 
not found.");
+Segments mergingSegs = 
cubeCopy.getMergingSegments(segCopy);
 
 Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment 
not exist.");
 
@@ -63,16 +63,16 @@ public class MergeOffsetStep extends AbstractExecutable {
 final CubeSegment first = mergingSegs.get(0);
 final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1);
 
-segment.setSegRange(new SegmentRange(first.getSegRange().start, 
last.getSegRange().end));
-
segment.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart());
-
segment.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd());
+segCopy.setSegRange(new SegmentRange(first.getSegRange().start, 
last.getSegRange().end));
+
segCopy.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart());
+
segCopy.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd());
 
-segment.setTSRange(new TSRange(mergingSegs.getTSStart(), 
mergingSegs.getTSEnd()));
+segCopy.setTSRange(new TSRange(mergingSegs.getTSStart(), 
mergingSegs.getTSEnd()));
 
-CubeUpdate cubeBuilder = new CubeUpdate(cube);
-cubeBuilder.setToUpdateSegs(segment);
+CubeUpdate update = new CubeUpdate(cubeCopy);
+update.setToUpdateSegs(segCopy);
 try {
-cubeManager.updateCube(cubeBuilder);
+cubeManager.updateCube(update);
 return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
 } catch (IOException e) {
 logger.error("fail to update cube segment offset", e);

[15/50] [abbrv] kylin git commit: minor, shorten PercentileAggregatorTest, taking too long

[liyang]

minor, shorten PercentileAggregatorTest, taking too long


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ff03c8c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ff03c8c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ff03c8c

Branch: refs/heads/master
Commit: 9ff03c8cf647256fa0c3ba3b7adc1947fe73cfe7
Parents: 03e6b8c
Author: Li Yang 
Authored: Sun Dec 10 17:52:29 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 14 16:53:10 2017 +0800

--
 .../apache/kylin/measure/percentile/PercentileAggregatorTest.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff03c8c/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
 
b/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
index 5fcd382..71e2b60 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/measure/percentile/PercentileAggregatorTest.java
@@ -60,7 +60,7 @@ public class PercentileAggregatorTest {
 @Test
 public void testLargeSize() throws Exception {
 for (int compression : compressions) {
-testPercentileSize(2000, null, compression);
+testPercentileSize(200, null, compression);
 }
 }
 

[39/50] [abbrv] kylin git commit: minor, refine metric.

[liyang]

minor, refine metric.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/826788a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/826788a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/826788a3

Branch: refs/heads/master
Commit: 826788a35b717159381e53adb3c5da16eeac360d
Parents: f2722eb
Author: tttMelody <245915...@qq.com>
Authored: Wed Dec 27 20:55:06 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 28 10:34:43 2017 +0800

--
 .../org/apache/kylin/rest/service/QueryService.java | 16 ++--
 1 file changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/826788a3/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 2b76ae3..9afec83 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -409,7 +409,6 @@ public class QueryService extends BasicService {
 HtraceInit.init();
 scope = Trace.startSpan("query life cycle for " + 
queryContext.getQueryId(), Sampler.ALWAYS);
 }
-
 String traceUrl = getTraceUrl(scope);
 
 try (SetThreadName ignored = new SetThreadName("Query %s", 
queryContext.getQueryId())) {
@@ -516,10 +515,11 @@ public class QueryService extends BasicService {
 
 sqlResponse.setTraceUrl(traceUrl);
 logQuery(sqlRequest, sqlResponse);
-
-QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
-QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
-
+try {
+recordMetric(sqlRequest, sqlResponse);
+} catch (Throwable th) {
+logger.warn("Write metric error.", th);
+}
 if (sqlResponse.getIsException())
 throw new 
InternalErrorException(sqlResponse.getExceptionMessage());
 
@@ -528,13 +528,17 @@ public class QueryService extends BasicService {
 } finally {
 BackdoorToggles.cleanToggles();
 QueryContext.reset();
-
 if (scope != null) {
 scope.close();
 }
 }
 }
 
+protected void recordMetric(SQLRequest sqlRequest, SQLResponse 
sqlResponse) throws UnknownHostException {
+QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
+QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
+}
+
 private String getTraceUrl(TraceScope scope) {
 if (scope == null) {
 return null;

[29/50] [abbrv] kylin git commit: KYLIN-3125 Support SparkSql in Cube building step 'Create Intermediate Flat Hive Table'

[liyang]

KYLIN-3125 Support SparkSql in Cube building step 'Create Intermediate Flat 
Hive Table'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/acc9d0c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/acc9d0c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/acc9d0c8

Branch: refs/heads/master
Commit: acc9d0c834bf48ede6c57e2cc7aee73b76531fa2
Parents: bcda327
Author: Li Yang 
Authored: Thu Dec 21 16:39:56 2017 +0800
Committer: 成 
Committed: Sat Dec 23 22:22:21 2017 +0800

--
 build/bin/find-hive-dependency.sh   |  3 ++-
 .../apache/kylin/common/KylinConfigBase.java| 16 ++
 .../kylin/common/util/HiveCmdBuilder.java   | 19 
 .../main/resources/kylin-defaults.properties|  9 
 .../kylin/common/util/HiveCmdBuilderTest.java   | 23 +++-
 .../apache/kylin/source/hive/HiveTableMeta.java |  4 ++--
 6 files changed, 66 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/acc9d0c8/build/bin/find-hive-dependency.sh
--
diff --git a/build/bin/find-hive-dependency.sh 
b/build/bin/find-hive-dependency.sh
index cb9c8f5..653db83 100755
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -29,8 +29,9 @@ hive_env=
 
 if [ "${client_mode}" == "beeline" ]
 then
+beeline_shell=`$KYLIN_HOME/bin/get-properties.sh 
kylin.source.hive.beeline-shell`
 beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh 
kylin.source.hive.beeline-params`
-hive_env=`beeline ${hive_conf_properties} ${beeline_params} 
--outputformat=dsv -e set 2>&1 | grep 'env:CLASSPATH' `
+hive_env=`${beeline_shell} ${hive_conf_properties} ${beeline_params} 
--outputformat=dsv -e set 2>&1 | grep 'env:CLASSPATH' `
 else
 hive_env=`hive ${hive_conf_properties} -e set 2>&1 | grep 'env:CLASSPATH'`
 fi

http://git-wip-us.apache.org/repos/asf/kylin/blob/acc9d0c8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5620ac0..93fbc23 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -691,10 +691,26 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.source.hive.client", "cli");
 }
 
+public String getHiveBeelineShell() {
+return getOptional("kylin.source.hive.beeline-shell", "beeline");
+}
+
 public String getHiveBeelineParams() {
 return getOptional("kylin.source.hive.beeline-params", "");
 }
 
+public boolean getEnableSparkSqlForTableOps() {
+return 
Boolean.parseBoolean(getOptional("kylin.source.hive.enable-sparksql-for-table-ops",
 "false"));
+}
+
+public String getSparkSqlBeelineShell() {
+return getOptional("kylin.source.hive.sparksql-beeline-shell", "");
+}
+
+public String getSparkSqlBeelineParams() {
+return getOptional("kylin.source.hive.sparksql-beeline-params", "");
+}
+
 public String getFlatHiveTableClusterByDictColumn() {
 return 
getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/acc9d0c8/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 77dc579..add53db 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -50,18 +50,28 @@ public class HiveCmdBuilder {
 CLI, BEELINE
 }
 
-private HiveClientMode clientMode;
 private KylinConfig kylinConfig;
 final private Map hiveConfProps = new HashMap<>();
 final private ArrayList statements = Lists.newArrayList();
 
 public HiveCmdBuilder() {
 kylinConfig = KylinConfig.getInstanceFromEnv();
-clientMode = 
HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase());
 loadHiveConfiguration();
 }
 
 public String build() {
+HiveClientMode clientMode = 
HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase());
+String beelineShell = 

[33/50] [abbrv] kylin git commit: #3314 fix bugs data size in first two steps

[liyang]

#3314  fix bugs data size in first two steps


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6e45a76
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6e45a76
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6e45a76

Branch: refs/heads/master
Commit: c6e45a76dac1f9428308622558fe3608e11b66df
Parents: 5aa8033
Author: Sheng Zhang 
Authored: Tue Dec 5 19:03:34 2017 +0800
Committer: Sheng Zhang 
Committed: Wed Dec 27 16:59:51 2017 +0800

--
 .../source/hive/CreateFlatHiveTableStep.java| 30 +++-
 .../apache/kylin/source/hive/HiveMRInput.java   |  8 ++
 2 files changed, 37 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c6e45a76/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index 48e7686..891b090 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -18,7 +18,14 @@
 package org.apache.kylin.source.hive;
 
 import java.io.IOException;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.HiveCmdBuilder;
 import org.apache.kylin.common.util.Pair;
@@ -26,6 +33,7 @@ import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
 import org.apache.kylin.job.common.PatternedLogger;
+import org.apache.kylin.job.constant.ExecutableConstants;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -39,6 +47,7 @@ public class CreateFlatHiveTableStep extends 
AbstractExecutable {
 
 private static final Logger logger = 
LoggerFactory.getLogger(CreateFlatHiveTableStep.class);
 protected final PatternedLogger stepLogger = new PatternedLogger(logger);
+private static final Pattern HDFS_LOCATION = Pattern.compile("LOCATION 
\'(.*)\';");
 
 protected void createFlatHiveTable(KylinConfig config) throws IOException {
 final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
@@ -51,12 +60,31 @@ public class CreateFlatHiveTableStep extends 
AbstractExecutable {
 stepLogger.log(cmd);
 
 Pair response = 
config.getCliCommandExecutor().execute(cmd, stepLogger);
-getManager().addJobInfo(getId(), stepLogger.getInfo());
+Map info = stepLogger.getInfo();
+
+//get the flat Hive table size
+Matcher matcher = HDFS_LOCATION.matcher(cmd);
+if (matcher.find()) {
+String hiveFlatTableHdfsUrl = matcher.group(1);
+long size = getFileSize(hiveFlatTableHdfsUrl);
+info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, "" + size);
+logger.info("HDFS_Bytes_Writen: " + size);
+}
+getManager().addJobInfo(getId(), info);
 if (response.getFirst() != 0) {
 throw new RuntimeException("Failed to create flat hive table, 
error code " + response.getFirst());
 }
 }
 
+private long getFileSize(String hdfsUrl) throws IOException {
+Configuration configuration = new Configuration();
+Path path = new Path(hdfsUrl);
+FileSystem fs = path.getFileSystem(configuration);
+ContentSummary contentSummary = fs.getContentSummary(path);
+long length = contentSummary.getLength();
+return length;
+}
+
 private KylinConfig getCubeSpecificConfig() {
 String cubeName = CubingExecutableUtil.getCubeName(getParams());
 CubeManager manager = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

http://git-wip-us.apache.org/repos/asf/kylin/blob/c6e45a76/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 3671266..0b23121 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ 

[20/50] [abbrv] kylin git commit: KYLIN-3108, Change the kafka config write path in zookeeper to /kylin/streaming_config/UUID.

[liyang]

KYLIN-3108, Change the kafka config write path in zookeeper to 
/kylin/streaming_config/UUID.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c856546
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c856546
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c856546

Branch: refs/heads/master
Commit: 9c8565465a7bc2d9b227900e3a6cf841e919d49e
Parents: f15da9e
Author: tttMelody <245915...@qq.com>
Authored: Thu Dec 14 16:23:22 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Thu Dec 14 19:08:11 2017 +0800

--
 .../org/apache/kylin/provision/BuildCubeWithStream.java | 12 +++-
 1 file changed, 7 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9c856546/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 60cea56..bf614af 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -90,7 +90,7 @@ public class BuildCubeWithStream {
 private KafkaConfig kafkaConfig;
 private MockKafka kafkaServer;
 private ZkConnection zkConnection;
-private final String kafkaZkPath = "/" + UUID.randomUUID().toString();
+private final String kafkaZkPath = "/kylin/streaming/" + 
UUID.randomUUID().toString();
 
 protected static boolean fastBuildMode = false;
 private volatile boolean generateData = true;
@@ -359,13 +359,15 @@ public class BuildCubeWithStream {
 buildCubeWithStream.before();
 buildCubeWithStream.build();
 logger.info("Build is done");
-
-buildCubeWithStream.after();
-buildCubeWithStream.cleanup();
-logger.info("Going to exit");
 } catch (Throwable e) {
 logger.error("error", e);
 exitCode = 1;
+} finally{
+if (buildCubeWithStream != null) {
+buildCubeWithStream.after();
+buildCubeWithStream.cleanup();
+}
+logger.info("Going to exit");
 }
 
 long millis = System.currentTimeMillis() - start;

[43/50] [abbrv] kylin git commit: Expose derived dimensions for get sql api

[liyang]

Expose derived dimensions for get sql api


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7889cec6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7889cec6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7889cec6

Branch: refs/heads/master
Commit: 7889cec6794a6dda7ba6151feb5e9f9b2378cb7e
Parents: a6d1ab0
Author: nichunen 
Authored: Thu Dec 28 21:36:13 2017 +0800
Committer: Li Yang 
Committed: Thu Dec 28 22:05:18 2017 -0600

--
 .../cube/model/CubeJoinedFlatTableDesc.java | 30 +++-
 .../kylin/rest/controller/CubeController.java   |  4 +--
 .../rest/controller/CubeControllerTest.java | 17 +++
 3 files changed, 42 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7889cec6/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index f8b039a..81087f8 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -44,23 +44,31 @@ public class CubeJoinedFlatTableDesc implements 
IJoinedFlatTableDesc, Serializab
 protected final String tableName;
 protected final CubeDesc cubeDesc;
 protected final CubeSegment cubeSegment;
+protected final boolean includingDerived;
 
 private int columnCount = 0;
 private List columnList = Lists.newArrayList();
 private Map columnIndexMap = Maps.newHashMap();
 
 public CubeJoinedFlatTableDesc(CubeDesc cubeDesc) {
-this(cubeDesc, null);
+this(cubeDesc, null, false);
+}
+
+public CubeJoinedFlatTableDesc(CubeDesc cubeDesc, boolean 
includingDerived) {
+this(cubeDesc, null, includingDerived);
 }
 
 public CubeJoinedFlatTableDesc(CubeSegment cubeSegment) {
-this(cubeSegment.getCubeDesc(), cubeSegment);
+this(cubeSegment.getCubeDesc(), cubeSegment, false);
 }
 
-private CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment 
/* can be null */) {
+private CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment 
/* can be null */,
+boolean includingDerived) {
 this.cubeDesc = cubeDesc;
 this.cubeSegment = cubeSegment;
+
 this.tableName = makeTableName(cubeDesc, cubeSegment);
+this.includingDerived = includingDerived;
 initParseCubeDesc();
 }
 
@@ -68,7 +76,8 @@ public class CubeJoinedFlatTableDesc implements 
IJoinedFlatTableDesc, Serializab
 if (cubeSegment == null) {
 return "kylin_intermediate_" + cubeDesc.getName().toLowerCase();
 } else {
-return "kylin_intermediate_" + cubeDesc.getName().toLowerCase() + 
"_" + cubeSegment.getUuid().replaceAll("-", "_");
+return "kylin_intermediate_" + cubeDesc.getName().toLowerCase() + 
"_"
++ cubeSegment.getUuid().replaceAll("-", "_");
 }
 }
 
@@ -86,8 +95,14 @@ public class CubeJoinedFlatTableDesc implements 
IJoinedFlatTableDesc, Serializab
 
 // check what columns from hive tables are required, and index them
 protected void initParseCubeDesc() {
-for (TblColRef col : 
cubeDesc.listDimensionColumnsExcludingDerived(false)) {
-initAddColumn(col);
+if (this.includingDerived) {
+for (TblColRef col : 
cubeDesc.listDimensionColumnsIncludingDerived()) {
+initAddColumn(col);
+}
+} else {
+for (TblColRef col : 
cubeDesc.listDimensionColumnsExcludingDerived(false)) {
+initAddColumn(col);
+}
 }
 
 List measures = cubeDesc.getMeasures();
@@ -118,7 +133,8 @@ public class CubeJoinedFlatTableDesc implements 
IJoinedFlatTableDesc, Serializab
 // sanity check the input record (in bytes) matches what's expected
 public void sanityCheck(BytesSplitter bytesSplitter) {
 if (columnCount != bytesSplitter.getBufferSize()) {
-throw new IllegalArgumentException("Expect " + columnCount + " 
columns, but see " + bytesSplitter.getBufferSize() + " -- " + bytesSplitter);
+throw new IllegalArgumentException("Expect " + columnCount + " 
columns, but see "
++ bytesSplitter.getBufferSize() + " -- " + bytesSplitter);
 }
 
 // TODO: check data types here

[49/50] [abbrv] kylin git commit: Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --cc core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 17e3aa5,e5ec2d7..1fbbbe6
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@@ -43,7 -45,7 +43,6 @@@ import org.apache.kylin.common.util.Dic
  import org.apache.kylin.common.util.Pair;
  import org.apache.kylin.cube.cuboid.Cuboid;
  import org.apache.kylin.cube.model.CubeDesc;
--import org.apache.kylin.cube.model.DictionaryDesc;
  import org.apache.kylin.dict.DictionaryInfo;
  import org.apache.kylin.dict.DictionaryManager;
  import org.apache.kylin.dict.lookup.LookupStringTable;
@@@ -378,632 -445,555 +378,694 @@@ public class CubeManager implements IRe
  return cube;
  }
  
 -// append a full build segment
 -public CubeSegment appendSegment(CubeInstance cube) throws IOException {
 -return appendSegment(cube, null, null, null, null);
 -}
 -
 -public CubeSegment appendSegment(CubeInstance cube, TSRange tsRange) 
throws IOException {
 -return appendSegment(cube, tsRange, null, null, null);
 +// for test
 +CubeInstance reloadCube(String cubeName) {
 +try (AutoLock lock = cubeMapLock.lockForWrite()) {
 +return crud.reload(cubeName);
 +}
  }
  
 -public CubeSegment appendSegment(CubeInstance cube, SourcePartition src) 
throws IOException {
 -return appendSegment(cube, src.getTSRange(), src.getSegRange(), 
src.getSourcePartitionOffsetStart(),
 -src.getSourcePartitionOffsetEnd());
 +// for internal
 +CubeInstance reloadCubeQuietly(String cubeName) {
 +try (AutoLock lock = cubeMapLock.lockForWrite()) {
 +CubeInstance cube = crud.reloadQuietly(cubeName);
 +if (cube != null)
 +Cuboid.clearCache(cube);
 +return cube;
 +}
  }
  
 -CubeSegment appendSegment(CubeInstance cube, TSRange tsRange, 
SegmentRange segRange,
 -Map sourcePartitionOffsetStart, Map 
sourcePartitionOffsetEnd)
 -throws IOException {
 -checkInputRanges(tsRange, segRange);
 -checkBuildingSegment(cube);
 -
 -// fix start/end a bit
 -if (cube.getModel().getPartitionDesc().isPartitioned()) {
 -// if missing start, set it to where last time ends
 -CubeSegment last = cube.getLastSegment();
 -CubeDesc cubeDesc = cube.getDescriptor();
 -if (tsRange != null && tsRange.start.v == 0) {
 -if (last == null) {
 -tsRange = new TSRange(cubeDesc.getPartitionDateStart(), 
tsRange.end.v);
 -} else if (!last.isOffsetCube()) {
 -tsRange = new TSRange(last.getTSRange().end.v, 
tsRange.end.v);
 +public void removeCubeLocal(String cubeName) {
 +try (AutoLock lock = cubeMapLock.lockForWrite()) {
 +CubeInstance cube = cubeMap.get(cubeName);
 +if (cube != null) {
 +cubeMap.removeLocal(cubeName);
 +for (CubeSegment segment : cube.getSegments()) {
 +usedStorageLocation.remove(segment.getUuid());
  }
 +Cuboid.clearCache(cube);
  }
 -} else {
 -// full build
 -tsRange = null;
 -segRange = null;
  }
 +}
  
 -CubeSegment newSegment = newSegment(cube, tsRange, segRange);
 -newSegment.setSourcePartitionOffsetStart(sourcePartitionOffsetStart);
 -newSegment.setSourcePartitionOffsetEnd(sourcePartitionOffsetEnd);
 -validateNewSegments(cube, newSegment);
 +public CubeInstance dropCube(String cubeName, boolean deleteDesc) throws 
IOException {
 +try (AutoLock lock = cubeMapLock.lockForWrite()) {
 +logger.info("Dropping cube '" + cubeName + "'");
 +// load projects before remove cube from project
  
 -CubeUpdate cubeBuilder = new CubeUpdate(cube);
 -cubeBuilder.setToAddSegs(newSegment);
 -updateCube(cubeBuilder);
 -return newSegment;
 -}
 +// delete cube instance and cube desc
 +CubeInstance cube = getCube(cubeName);
  
 -public CubeSegment refreshSegment(CubeInstance cube, TSRange tsRange, 
SegmentRange segRange) throws IOException {
 -checkInputRanges(tsRange, segRange);
 -checkBuildingSegment(cube);
 -
 -CubeSegment newSegment = newSegment(cube, tsRange, segRange);
 -
 -Pair pair = 
cube.getSegments().fitInSegments(newSegment);
 -if (pair.getFirst() == false || pair.getSecond() == false)
 -throw new IllegalArgumentException("The new refreshing segment " 
+ 

[08/50] [abbrv] kylin git commit: KYLIN-3090 Refactor to consolidate all caches and managers under KylinConfig

[liyang]

KYLIN-3090 Refactor to consolidate all caches and managers under KylinConfig


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8d79870
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8d79870
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8d79870

Branch: refs/heads/master
Commit: b8d7987090d96a763569a20a3f1665a3a612b565
Parents: bace2d2
Author: Li Yang 
Authored: Thu Dec 7 19:10:34 2017 +0800
Committer: Dong Li 
Committed: Mon Dec 11 10:16:35 2017 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  43 ++-
 .../apache/kylin/common/KylinConfigBase.java|   4 +
 .../kylin/common/persistence/ResourceStore.java |   4 +
 .../persistence/RootPersistentEntity.java   |   9 +
 .../kylin/common/util/AutoReadWriteLock.java|  66 
 .../common/util/AbstractKylinTestCase.java  |  34 --
 .../common/util/AutoReadWriteLockTest.java  |  43 +++
 .../org/apache/kylin/cube/CubeDescManager.java  |  51 +--
 .../java/org/apache/kylin/cube/CubeManager.java |  38 +--
 .../org/apache/kylin/cube/cuboid/Cuboid.java|  77 ++---
 .../apache/kylin/cube/cuboid/CuboidManager.java |  76 +
 .../upgrade/common/CubeMetadataUpgrade.java |  10 +-
 .../kylin/cube/AggregationGroupRuleTest.java|   2 -
 .../org/apache/kylin/cube/CubeDescTest.java |   4 +-
 .../apache/kylin/cube/CubeManagerCacheTest.java |   5 -
 .../kylin/cube/common/RowKeySplitterTest.java   |   2 -
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |   4 +-
 .../apache/kylin/cube/cuboid/CuboidTest.java|   2 -
 .../apache/kylin/cube/kv/RowKeyDecoderTest.java |   2 -
 .../apache/kylin/cube/kv/RowKeyEncoderTest.java |   2 -
 .../apache/kylin/dict/DictionaryManager.java|  44 +--
 .../kylin/dict/lookup/SnapshotManager.java  |  46 +--
 .../org/apache/kylin/job/dao/ExecutableDao.java |  34 +-
 .../kylin/job/execution/ExecutableManager.java  |  36 +-
 .../impl/threadpool/DistributedScheduler.java   |  45 +--
 .../kylin/metadata/TableMetadataManager.java|  58 +---
 .../kylin/metadata/TempStatementManager.java|  48 +--
 .../kylin/metadata/acl/TableACLManager.java |  41 +--
 .../badquery/BadQueryHistoryManager.java|  42 +--
 .../kylin/metadata/cachesync/Broadcaster.java   |  40 +--
 .../metadata/cachesync/CachedCrudAssist.java| 139 
 .../kylin/metadata/draft/DraftManager.java  |  32 +-
 .../kylin/metadata/model/DataModelManager.java  |  47 +--
 .../kylin/metadata/project/ProjectInstance.java |   5 +
 .../kylin/metadata/project/ProjectManager.java  | 334 ++-
 .../realization/RealizationRegistry.java|  38 +--
 .../metadata/streaming/StreamingManager.java|  60 +---
 .../gtrecord/GTCubeStorageQueryBase.java|   2 +-
 .../kylin/storage/hybrid/HybridManager.java |  37 +-
 .../engine/mr/steps/MergeCuboidMapperTest.java  |   8 +-
 .../rest/controller/ProjectController.java  |   3 +-
 .../java/org/apache/kylin/rest/msg/Message.java |   4 +
 .../kylin/rest/service/ProjectService.java  |  25 --
 .../rest/service/DiagnosisServiceTest.java  |   2 -
 .../rest/controller/ProjectControllerTest.java  |  20 +-
 .../kylin/source/kafka/KafkaConfigManager.java  |  54 +--
 .../hbase/util/ExtendCubeToHybridCLI.java   |  11 +-
 .../cube/MeasureTypeOnlyAggrInBaseTest.java |   6 +-
 .../hbase/steps/RowValueDecoderTest.java|   2 -
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  11 +-
 .../apache/kylin/tool/KylinLogExtractor.java|   6 +-
 .../apache/kylin/tool/CubeMetaIngesterTest.java |  10 +-
 52 files changed, 686 insertions(+), 1082 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index d014262..c638ec6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -26,11 +26,13 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.StringReader;
+import java.lang.reflect.Method;
 import java.net.URL;
 import java.nio.ByteOrder;
 import java.nio.charset.Charset;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
@@ -382,6 +384,8 @@ public class KylinConfig extends KylinConfigBase {
 }
 
 // 

+
+Map

[34/50] [abbrv] kylin git commit: minor, restClient support set http timeout

[liyang]

minor, restClient support set http timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/63de91df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/63de91df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/63de91df

Branch: refs/heads/master
Commit: 63de91df546c28c4aa745f61619eeb8797eb7b44
Parents: c6e45a7
Author: Chao Long 
Authored: Wed Dec 27 18:12:19 2017 +0800
Committer: Dong Li 
Committed: Wed Dec 27 18:58:35 2017 +0800

--
 .../kylin/common/restclient/RestClient.java | 27 +---
 .../kylin/rest/controller/BasicController.java  |  2 +-
 2 files changed, 24 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/63de91df/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java 
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 67ed9c1..02045ae 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -61,8 +61,8 @@ public class RestClient {
 
 protected static Pattern fullRestPattern = 
Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
 
-private static final int HTTP_CONNECTION_TIMEOUT_MS = 3;
-private static final int HTTP_SOCKET_TIMEOUT_MS = 12;
+private int httpConnectionTimeoutMs = 3;
+private int httpSocketTimeoutMs = 12;
 
 public static final String SCHEME_HTTP = "http://;;
 
@@ -86,6 +86,10 @@ public class RestClient {
  * @param uri "user:pwd@host:port"
  */
 public RestClient(String uri) {
+this(uri, null, null);
+}
+
+public RestClient(String uri, Integer httpConnectionTimeoutMs, Integer 
httpSocketTimeoutMs) {
 Matcher m = fullRestPattern.matcher(uri);
 if (!m.matches())
 throw new IllegalArgumentException("URI: " + uri + " -- does not 
match pattern " + fullRestPattern);
@@ -96,10 +100,25 @@ public class RestClient {
 String portStr = m.group(4);
 int port = Integer.parseInt(portStr == null ? "7070" : portStr);
 
+if (httpConnectionTimeoutMs != null)
+this.httpConnectionTimeoutMs = httpConnectionTimeoutMs;
+if (httpSocketTimeoutMs != null)
+this.httpSocketTimeoutMs = httpSocketTimeoutMs;
+
 init(host, port, user, pwd);
 }
 
 public RestClient(String host, int port, String userName, String password) 
{
+this(host, port, userName, password, null, null);
+}
+
+public RestClient(String host, int port, String userName, String password, 
Integer httpConnectionTimeoutMs,
+Integer httpSocketTimeoutMs) {
+if (httpConnectionTimeoutMs != null)
+this.httpConnectionTimeoutMs = httpConnectionTimeoutMs;
+if (httpSocketTimeoutMs != null)
+this.httpSocketTimeoutMs = httpSocketTimeoutMs;
+
 init(host, port, userName, password);
 }
 
@@ -111,8 +130,8 @@ public class RestClient {
 this.baseUrl = SCHEME_HTTP + host + ":" + port + KYLIN_API_PATH;
 
 final HttpParams httpParams = new BasicHttpParams();
-HttpConnectionParams.setSoTimeout(httpParams, HTTP_SOCKET_TIMEOUT_MS);
-HttpConnectionParams.setConnectionTimeout(httpParams, 
HTTP_CONNECTION_TIMEOUT_MS);
+HttpConnectionParams.setSoTimeout(httpParams, httpSocketTimeoutMs);
+HttpConnectionParams.setConnectionTimeout(httpParams, 
httpConnectionTimeoutMs);
 
 final PoolingClientConnectionManager cm = new 
PoolingClientConnectionManager();
 KylinConfig config = KylinConfig.getInstanceFromEnv();

http://git-wip-us.apache.org/repos/asf/kylin/blob/63de91df/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
index b77309e..539a64c 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
@@ -109,7 +109,7 @@ public class BasicController {
 
 protected void setDownloadResponse(String downloadFile, final 
HttpServletResponse response) {
 File file = new File(downloadFile);
-try (InputStream fileInputStream = new FileInputStream(file); 
OutputStream output = response.getOutputStream();) {
+try 

[23/50] [abbrv] kylin git commit: minor, refactor on JoinTree

[liyang]

minor, refactor on JoinTree


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9f603d31
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9f603d31
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9f603d31

Branch: refs/heads/master
Commit: 9f603d3188bd20070ea10cbe5fc0eecc28703633
Parents: 26e191a
Author: lidongsjtu 
Authored: Thu Nov 16 23:58:25 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 20 10:09:10 2017 +0800

--
 .../org/apache/kylin/job/JoinedFlatTable.java   |  2 +-
 .../kylin/metadata/model/DataModelDesc.java |  5 +-
 .../apache/kylin/metadata/model/JoinDesc.java   | 28 ---
 .../apache/kylin/metadata/model/JoinsTree.java  | 50 +++-
 4 files changed, 53 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9f603d31/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index d731d4f..d918777 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -178,7 +178,7 @@ public class JoinedFlatTable {
 if (i > 0) {
 sql.append(" AND ");
 }
-sql.append(fk[i].getIdentity() + " = " + 
pk[i].getIdentity());
+sql.append(fk[i].getExpressionInSourceDB() + " = " + 
pk[i].getExpressionInSourceDB());
 }
 sql.append(sep);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9f603d31/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index bf0db73..4dd4399 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -404,11 +404,12 @@ public class DataModelDesc extends RootPersistentEntity {
 alias = alias.toUpperCase();
 join.setAlias(alias);
 
-TableRef ref = new TableRef(this, alias, tableDesc, true);
+boolean isLookup = join.getKind() == TableKind.LOOKUP;
+TableRef ref = new TableRef(this, alias, tableDesc, isLookup);
 
 join.setTableRef(ref);
 addAlias(ref);
-(join.getKind() == TableKind.LOOKUP ? lookupTableRefs : 
factTableRefs).add(ref);
+(isLookup ? lookupTableRefs : factTableRefs).add(ref);
 }
 
 tableNameMap.putAll(aliasMap);

http://git-wip-us.apache.org/repos/asf/kylin/blob/9f603d31/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index b16ad7d..4bf9af3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -187,34 +187,6 @@ public class JoinDesc implements Serializable {
 return true;
 }
 
-// equals() without alias
-public boolean matches(JoinDesc other) {
-if (other == null)
-return false;
-
-if (!this.type.equalsIgnoreCase(other.getType()))
-return false;
-
-// note pk/fk are sorted, sortByFK()
-if (!this.columnDescEquals(foreignKeyColumns, other.foreignKeyColumns))
-return false;
-if (!this.columnDescEquals(primaryKeyColumns, other.primaryKeyColumns))
-return false;
-
-return true;
-}
-
-private boolean columnDescEquals(TblColRef[] a, TblColRef[] b) {
-if (a.length != b.length)
-return false;
-
-for (int i = 0; i < a.length; i++) {
-if (a[i].getColumnDesc().equals(b[i].getColumnDesc()) == false)
-return false;
-}
-return true;
-}
-
 @Override
 public String toString() {
 return "JoinDesc [type=" + type + ", primary_key=" + 
Arrays.toString(primaryKey) + ", foreign_key=" + Arrays.toString(foreignKey) + 
"]";

[44/50] [abbrv] kylin git commit: Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/webapp/app/js/controllers/cubeMeasures.js
--

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/webapp/app/js/controllers/cubeSchema.js
--
diff --cc webapp/app/js/controllers/cubeSchema.js
index 2a47a8c,0af7cc2..5eed9b9
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@@ -160,7 -160,24 +160,23 @@@ KylinApp.controller('CubeSchemaCtrl', f
  
  });
  
+ var queryParam = {offset: 0, limit: 65535};
+ 
+ CubeService.list(queryParam, function (all_cubes) {
+   if($scope.allCubes.length > 0){
+ $scope.allCubes.splice(0,$scope.allCubes.length);
+   }
+ 
+   for (var i = 0; i < all_cubes.length; i++) {
+ $scope.allCubes.push(all_cubes[i].name.toUpperCase());
+   }
+ });
+ 
+ // ~ public methods
+ $scope.filterProj = function(project){
+ return $scope.userService.hasRole('ROLE_ADMIN') || 
$scope.hasPermission(project,$scope.permissions.ADMINISTRATION.mask);
+ };
  
 -
  $scope.removeElement = function (arr, element) {
  var index = arr.indexOf(element);
  if (index > -1) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/webapp/app/js/controllers/modelSchema.js
--

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/webapp/app/js/controllers/page.js
--

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/webapp/app/js/controllers/query.js
--
diff --cc webapp/app/js/controllers/query.js
index 1c03684,f29cf58..ea45d88
--- a/webapp/app/js/controllers/query.js
+++ b/webapp/app/js/controllers/query.js
@@@ -21,8 -21,10 +21,11 @@@
  KylinApp
  .controller('QueryCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, $modal, MessageService, 
$domUtilityService, $timeout, TableService, SweetAlert, VdmUtil) {
  $scope.mainPanel = 'query';
+ if ($routeParams.queryPanel) {
+ $scope.mainPanel = $routeParams.queryPanel;
+ }
  $scope.rowsPerPage = 5;
 +$scope.hasLimit = true;
  $scope.base64 = $base64;
  $scope.queryString = "";
  $scope.queries = [];
@@@ -433,10 -423,7 +432,9 @@@
  
  
  })
 -.controller('QueryResultCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, queryConfig) {
 +.controller('QueryResultCtrl', function ($scope, storage, $base64, $q, 
$location, $anchorScroll, $routeParams, QueryService, GraphService, 
kylinConfig) {
 +$scope.isAdminExportAllowed = kylinConfig.isAdminExportAllowed();
 +$scope.isNonAdminExportAllowed = 
kylinConfig.isNonAdminExportAllowed();
- 
  $scope.buildGraphMetadata = function (query) {
  if (!query.graph.show) {
  return;

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/webapp/app/js/services/kylinProperties.js
--
diff --cc webapp/app/js/services/kylinProperties.js
index 76d7763,e31dd6c..dc9b94d
--- a/webapp/app/js/services/kylinProperties.js
+++ b/webapp/app/js/services/kylinProperties.js
@@@ -114,22 -114,22 +114,38 @@@ KylinApp.service('kylinConfig', functio
  }
  return true;
}
- 
++  
 +  this.isAdminExportAllowed = function(){
 +var status = this.getProperty("kylin.web.export-allow-admin").trim();
 +if(status!=='false'){
 +  return true;
 +}
 +return false;
 +  }
  
 +  this.isNonAdminExportAllowed = function(){
 +var status = this.getProperty("kylin.web.export-allow-other").trim();
 +if(status!=='false'){
 +  return true;
 +}
 +return false;
 +  }
++  
+   this.getHiddenMeasures = function() {
+ var hide_measures = 
this.getProperty("kylin.web.hide-measures").replace(/\s/g,"").toUpperCase();
+ return hide_measures.split(",")
+   }
+ 
+   this.getQueryTimeout = function () {
+ var queryTimeout = parseInt(this.getProperty("kylin.web.query-timeout"));
+ if (isNaN(queryTimeout)) {
+queryTimeout = 30;
+ }
+ return queryTimeout;
+   }
  
+   this.isInitialized = function() {
+ return angular.isString(_config);
 -  }  
++  }
  });
  

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/webapp/app/partials/cubes/cube_detail.html
--
diff --cc webapp/app/partials/cubes/cube_detail.html
index 43772b4,38806e1..db1a874
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@@ -26,15 -26,15 +26,15 @@@
  

[21/50] [abbrv] kylin git commit: KYLIN-3105, remove Interface#Scheduler's stop method.

[liyang]

KYLIN-3105, remove Interface#Scheduler's stop method.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7ebaec5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7ebaec5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7ebaec5

Branch: refs/heads/master
Commit: d7ebaec5162d8714ba8c2b86249d4b83cef5fdbe
Parents: 9c85654
Author: tttMelody <245915...@qq.com>
Authored: Thu Dec 14 16:44:05 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Thu Dec 14 19:51:54 2017 +0800

--
 .../src/main/java/org/apache/kylin/job/Scheduler.java |  2 --
 .../kylin/job/impl/threadpool/DefaultScheduler.java   | 10 --
 .../kylin/job/impl/threadpool/DistributedScheduler.java   | 10 --
 .../apache/kylin/job/impl/threadpool/NoopScheduler.java   |  5 -
 4 files changed, 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d7ebaec5/core-job/src/main/java/org/apache/kylin/job/Scheduler.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/Scheduler.java 
b/core-job/src/main/java/org/apache/kylin/job/Scheduler.java
index 93d2510..08ec771 100644
--- a/core-job/src/main/java/org/apache/kylin/job/Scheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/Scheduler.java
@@ -31,8 +31,6 @@ public interface Scheduler {
 
 void shutdown() throws SchedulerException;
 
-boolean stop(T executable) throws SchedulerException;
-
 boolean hasStarted();
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d7ebaec5/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 327a793..78f10bf 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -249,16 +249,6 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 }
 
 @Override
-public boolean stop(AbstractExecutable executable) throws 
SchedulerException {
-if (hasStarted) {
-return true;
-} else {
-//TODO should try to stop this executable
-return true;
-}
-}
-
-@Override
 public boolean hasStarted() {
 return this.hasStarted;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d7ebaec5/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index 2c934d0..055de4d 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -340,16 +340,6 @@ public class DistributedScheduler implements 
Scheduler, Conn
 }
 
 @Override
-public boolean stop(AbstractExecutable executable) throws 
SchedulerException {
-if (hasStarted) {
-return true;
-} else {
-//TODO should try to stop this executable
-return true;
-}
-}
-
-@Override
 public boolean hasStarted() {
 return this.hasStarted;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d7ebaec5/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/NoopScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/NoopScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/NoopScheduler.java
index 0a7eecf..2e25611 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/NoopScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/NoopScheduler.java
@@ -40,11 +40,6 @@ public class NoopScheduler implements 
Scheduler {
 }
 
 @Override
-public boolean stop(AbstractExecutable executable) throws 
SchedulerException {
-return false;
-}
-
-@Override
 public boolean hasStarted() {
 return false;
 }

[38/50] [abbrv] kylin git commit: KYLIN-3106 DefaultScheduler.shutdown should use ExecutorService.shutdownNow instead of ExecutorService.shutdown

[liyang]

KYLIN-3106 DefaultScheduler.shutdown should use ExecutorService.shutdownNow 
instead of ExecutorService.shutdown

fix ui


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f2722eb7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f2722eb7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f2722eb7

Branch: refs/heads/master
Commit: f2722eb7d147735c79102e9e4bb35bf7b38d8534
Parents: 3b200a7
Author: Hongbin Ma 
Authored: Thu Dec 14 14:58:50 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 27 20:05:54 2017 +0800

--
 .../kylin/job/execution/ExecutableManager.java  |  5 +
 .../job/impl/threadpool/DefaultScheduler.java   | 15 ++-
 .../job/FiveSecondSucceedTestExecutable.java| 46 +
 .../job/impl/threadpool/BaseSchedulerTest.java  | 39 +---
 .../impl/threadpool/DefaultSchedulerTest.java   | 99 +++-
 5 files changed, 160 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f2722eb7/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 6110573..561c38d 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -390,6 +390,11 @@ public class ExecutableManager {
 }
 
 public void updateJobOutput(String jobId, ExecutableState newStatus, 
Map info, String output) {
+// when 
+if (Thread.currentThread().isInterrupted()) {
+throw new RuntimeException("Current thread is interruptted, 
aborting");
+}
+
 try {
 final ExecutableOutputPO jobOutput = 
executableDao.getJobOutput(jobId);
 Preconditions.checkArgument(jobOutput != null, "there is no 
related output for job id:" + jobId);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2722eb7/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 42185cc..69d85ca 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -81,7 +81,8 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 
 @Override
 synchronized public void run() {
-try {
+try (SetThreadName ignored = new SetThreadName("Scheduler %s 
FetcherRunner",
+System.identityHashCode(DefaultScheduler.this))) {
 // logger.debug("Job Fetcher is running...");
 Map runningJobs = context.getRunningJobs();
 if (isJobPoolFull()) {
@@ -141,7 +142,7 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 + nStopped + " stopped, " + nReady + " ready, " + 
nSUCCEED + " already succeed, " + nError
 + " error, " + nDiscarded + " discarded, " + nOthers + 
" others");
 } catch (Exception e) {
-fetchFailed = true;
+fetchFailed = true; // this could happen when resource store 
is unavailable
 logger.warn("Job Fetcher caught a exception ", e);
 }
 }
@@ -258,12 +259,18 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 logger.info("Shutting down DefaultScheduler ");
 jobLock.unlockJobEngine();
 try {
-fetcherPool.shutdown();
+fetcherPool.shutdownNow();//interrupt
 fetcherPool.awaitTermination(1, TimeUnit.MINUTES);
-jobPool.shutdown();
+} catch (InterruptedException e) {
+//ignore it
+logger.warn("InterruptedException is caught!");
+}
+try {
+jobPool.shutdownNow();//interrupt
 jobPool.awaitTermination(1, TimeUnit.MINUTES);
 } catch (InterruptedException e) {
 //ignore it
+logger.warn("InterruptedException is caught!");
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2722eb7/core-job/src/test/java/org/apache/kylin/job/FiveSecondSucceedTestExecutable.java
--
diff --git 

[13/50] [abbrv] kylin git commit: minor, skip column family ordering check for compatibility

[liyang]

minor, skip column family ordering check for compatibility


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/05610dee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/05610dee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/05610dee

Branch: refs/heads/master
Commit: 05610dee7990d819a615efe57e8f9782948ee36f
Parents: 1e35990
Author: Roger Shi 
Authored: Wed Dec 13 12:04:02 2017 +0800
Committer: Roger Shi 
Committed: Wed Dec 13 14:16:02 2017 +0800

--
 .../java/org/apache/kylin/cube/model/CubeDesc.java|  7 +--
 .../test/java/org/apache/kylin/cube/CubeDescTest.java | 14 --
 2 files changed, 5 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/05610dee/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index efb9c59..32578ed 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -1039,8 +1039,11 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 checkState(!measureSet.contains(colMeasureRefs[i]), 
"measure (%s) duplicates", colMeasureRefs[i]);
 measureSet.add(colMeasureRefs[i]);
 
-checkState(measureIndex[i] > lastMeasureIndex, "measure 
(%s) is not in order", colMeasureRefs[i]);
-lastMeasureIndex = measureIndex[i];
+if (storageType > IStorageAware.ID_SHARDED_HBASE) {
+checkState(measureIndex[i] > lastMeasureIndex, 
"measure (%s) is not in order",
+colMeasureRefs[i]);
+lastMeasureIndex = measureIndex[i];
+}
 
 checkEachMeasureExist.set(measureIndex[i]);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/05610dee/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
--
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 36ac5c3..29152fe 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -322,20 +322,6 @@ public class CubeDescTest extends 
LocalFileMetadataTestCase {
 }
 
 @Test
-public void testBadInit13() throws Exception {
-thrown.expect(IllegalStateException.class);
-CubeDesc cubeDesc = 
CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
-MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
-Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
-thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") 
is not in order");
-HBaseColumnDesc colDesc = new HBaseColumnDesc();
-colDesc.setQualifier("M");
-colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", 
"ITEM_COUNT_SUM", "TRANS_CNT" });
-cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = 
colDesc;
-cubeDesc.initMeasureReferenceToColumnFamily();
-}
-
-@Test
 public void testBadInit14() throws Exception {
 thrown.expect(IllegalStateException.class);
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);

[10/50] [abbrv] kylin git commit: Fix wrong error message for ArithmeticException

[liyang]

Fix wrong error message for ArithmeticException


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af150625
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af150625
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af150625

Branch: refs/heads/master
Commit: af15062543b93c8146ae75bbb90f971e86a6bf96
Parents: 15dd1c1
Author: nichunen 
Authored: Mon Dec 11 11:07:34 2017 +0800
Committer: Ni Chunen 
Committed: Tue Dec 12 12:57:18 2017 +0800

--
 query/src/main/java/org/apache/kylin/query/util/QueryUtil.java | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/af150625/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java 
b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 3d21818..24d0c06 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -104,6 +104,11 @@ public class QueryUtil {
 msg = cause.getMessage();
 break;
 }
+
+if (cause.getClass().getName().contains("ArithmeticException")) {
+msg = "ArithmeticException: " + cause.getMessage();
+break;
+}
 cause = cause.getCause();
 }
 

[35/50] [abbrv] kylin git commit: Revert "minor, change the way that get DefaultScheduler's instance."

[liyang]

Revert "minor, change the way that get DefaultScheduler's instance."

This reverts commit 6cf61248f93f57bc6aa8b01f8898de3eb0bf3d3c.

fix ut


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6c31ca92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6c31ca92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6c31ca92

Branch: refs/heads/master
Commit: 6c31ca9235a84df588806bc51b108d921db18e1e
Parents: 63de91d
Author: Hongbin Ma 
Authored: Sat Dec 23 20:23:19 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 27 20:05:54 2017 +0800

--
 .../job/impl/threadpool/DefaultScheduler.java   | 34 +++-
 .../job/impl/threadpool/BaseSchedulerTest.java  | 15 +
 .../kylin/provision/BuildCubeWithEngine.java|  3 +-
 .../kylin/provision/BuildCubeWithStream.java|  3 +-
 4 files changed, 38 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6c31ca92/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index c8ab7ea..42185cc 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -62,14 +62,21 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 volatile boolean fetchFailed = false;
 private JobEngineConfig jobEngineConfig;
 
-private static volatile DefaultScheduler INSTANCE = null;
+private static DefaultScheduler INSTANCE = null;
 
 public DefaultScheduler() {
 if (INSTANCE != null) {
-throw new IllegalStateException("DefaultScheduler has been 
initiated. Use getInstance() instead.");
+throw new IllegalStateException("DefaultScheduler has been 
initiated.");
 }
 }
 
+public static DefaultScheduler getInstance() {
+if (INSTANCE == null) {
+INSTANCE = createInstance();
+}
+return INSTANCE;
+}
+
 private class FetcherRunner implements Runnable {
 
 @Override
@@ -170,7 +177,7 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 } finally {
 context.removeRunningJob(executable);
 }
-
+
 // trigger the next step asap
 fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
 }
@@ -180,7 +187,6 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 public void stateChanged(CuratorFramework client, ConnectionState 
newState) {
 if ((newState == ConnectionState.SUSPENDED) || (newState == 
ConnectionState.LOST)) {
 try {
-logger.info("ZK Connection state change to " + newState + ", 
shutdown default scheduler.");
 shutdown();
 } catch (SchedulerException e) {
 throw new RuntimeException("failed to shutdown scheduler", e);
@@ -188,13 +194,25 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 }
 }
 
-public synchronized static DefaultScheduler getInstance() {
-if (INSTANCE == null) {
-INSTANCE = new DefaultScheduler();
-}
+public synchronized static DefaultScheduler createInstance() {
+destroyInstance();
+INSTANCE = new DefaultScheduler();
 return INSTANCE;
 }
 
+public synchronized static void destroyInstance() {
+DefaultScheduler tmp = INSTANCE;
+INSTANCE = null;
+if (tmp != null) {
+try {
+tmp.shutdown();
+} catch (SchedulerException e) {
+logger.error("error stop DefaultScheduler", e);
+throw new RuntimeException(e);
+}
+}
+}
+
 @Override
 public synchronized void init(JobEngineConfig jobEngineConfig, JobLock 
lock) throws SchedulerException {
 jobLock = lock;

http://git-wip-us.apache.org/repos/asf/kylin/blob/6c31ca92/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index 6786d31..e0a542e 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ 

[32/50] [abbrv] kylin git commit: minor, add ut for load hive table.

[liyang]

minor, add ut for load hive table.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5aa80335
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5aa80335
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5aa80335

Branch: refs/heads/master
Commit: 5aa8033525c9223d305b233928e29ee32193715a
Parents: 19c4eb7
Author: tttMelody <245915...@qq.com>
Authored: Tue Dec 26 11:14:57 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 27 15:05:12 2017 +0800

--
 .../apache/kylin/rest/service/TableService.java | 55 +++-
 .../kylin/rest/service/TableServiceTest.java| 45 
 2 files changed, 75 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5aa80335/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index 6bb446c..d218bc2 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -76,10 +76,6 @@ public class TableService extends BasicService {
 private ModelService modelService;
 
 @Autowired
-@Qualifier("projectService")
-private ProjectService projectService;
-
-@Autowired
 @Qualifier("streamingMgmtService")
 private StreamingService streamingService;
 
@@ -115,27 +111,11 @@ public class TableService extends BasicService {
 
 public String[] loadHiveTablesToProject(String[] tables, String project) 
throws Exception {
 aclEvaluate.checkProjectAdminPermission(project);
-// de-dup
-SetMultimap db2tables = LinkedHashMultimap.create();
-for (String fullTableName : tables) {
-String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
-db2tables.put(parts[0], parts[1]);
-}
+List> allMeta = getAllMeta(tables, 
project);
+return loadHiveTablesToProject(project, allMeta);
+}
 
-// load all tables first
-List> allMeta = Lists.newArrayList();
-ISourceMetadataExplorer explr = 
SourceFactory.getDefaultSource().getSourceMetadataExplorer();
-for (Map.Entry entry : db2tables.entries()) {
-Pair pair = 
explr.loadTableMetadata(entry.getKey(), entry.getValue(), project);
-TableDesc tableDesc = pair.getFirst();
-
Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase()));
-
Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase()));
-
Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase()
 + "." + entry
-.getValue().toUpperCase()));
-TableExtDesc extDesc = pair.getSecond();
-
Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity()));
-allMeta.add(pair);
-}
+String[] loadHiveTablesToProject(String project, List> allMeta) throws Exception {
 
 // do schema check
 TableMetadataManager metaMgr = getTableManager();
@@ -181,7 +161,32 @@ public class TableService extends BasicService {
 addTableToProject(result, project);
 return result;
 }
-
+
+private List> getAllMeta(String[] tables, 
String project) throws Exception {
+// de-dup
+SetMultimap db2tables = LinkedHashMultimap.create();
+for (String fullTableName : tables) {
+String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
+db2tables.put(parts[0], parts[1]);
+}
+
+// load all tables first
+List> allMeta = Lists.newArrayList();
+ISourceMetadataExplorer explr = 
SourceFactory.getDefaultSource().getSourceMetadataExplorer();
+for (Map.Entry entry : db2tables.entries()) {
+Pair pair = 
explr.loadTableMetadata(entry.getKey(), entry.getValue(), project);
+TableDesc tableDesc = pair.getFirst();
+
Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase()));
+
Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase()));
+
Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase()
 + "." + entry
+

[05/50] [abbrv] kylin git commit: KYLIN-3092 Synchronize w/r operations on entity-caching managers

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f39432b..efb9c59 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -47,6 +47,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.KylinVersion;
+import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.Array;
@@ -58,13 +59,13 @@ import 
org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.DataModelManager;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -90,6 +91,11 @@ import com.google.common.collect.Sets;
 public class CubeDesc extends RootPersistentEntity implements IEngineAware {
 private static final Logger logger = 
LoggerFactory.getLogger(CubeDesc.class);
 
+// Use with care! Normally you should go to CubeDescManager and don't need 
this.
+public static JsonSerializer newSerializerForLowLevelAccess() {
+return new JsonSerializer<>(CubeDesc.class);
+}
+
 public static class CannotFilterExtendedColumnException extends 
RuntimeException {
 public CannotFilterExtendedColumnException(TblColRef tblColRef) {
 super(tblColRef == null ? "null" : tblColRef.getCanonicalName());
@@ -122,6 +128,8 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 }
 
 }
+
+// 

 
 private KylinConfigExt config;
 private DataModelDesc model;
@@ -183,6 +191,9 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 @JsonInclude(JsonInclude.Include.NON_NULL)
 private int parentForward = 3;
 
+// Error messages during resolving json metadata
+private List errors = new ArrayList();
+
 private LinkedHashSet allColumns = new LinkedHashSet<>();
 private LinkedHashSet allColumnDescs = new LinkedHashSet<>();
 private LinkedHashSet dimensionColumns = new LinkedHashSet<>();
@@ -194,6 +205,11 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 transient volatile private CuboidScheduler cuboidScheduler = null;
 
+@Override
+public String resourceName() {
+return name;
+}
+
 public boolean isEnableSharding() {
 //in the future may extend to other storage that is shard-able
 return storageType != IStorageAware.ID_HBASE && storageType != 
IStorageAware.ID_HYBRID;
@@ -204,11 +220,6 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 }
 
 /**
- * Error messages during resolving json metadata
- */
-private List errors = new ArrayList();
-
-/**
  * @return all columns this cube can support, including derived
  */
 public Set listAllColumns() {
@@ -311,7 +322,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 }
 
 public String getResourcePath() {
-return concatResourcePath(name);
+return concatResourcePath(resourceName());
 }
 
 public static String concatResourcePath(String descName) {
@@ -1116,6 +1127,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) {
 this.autoMergeTimeRanges = autoMergeTimeRanges;
 }
+
+public boolean isBroken() {
+return !errors.isEmpty();
+}
 
 public void addError(String message) {
 this.errors.add(message);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
--
diff 

[09/50] [abbrv] kylin git commit: minor, print the exception in log

[liyang]

minor, print the exception in log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/15dd1c12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/15dd1c12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/15dd1c12

Branch: refs/heads/master
Commit: 15dd1c12bc9b64981e14a6dbe2a28cd987acb588
Parents: f2f487f
Author: Cheng Wang 
Authored: Mon Dec 11 16:47:12 2017 +0800
Committer: Li Yang 
Committed: Mon Dec 11 02:56:17 2017 -0600

--
 .../org/apache/kylin/job/impl/threadpool/DefaultScheduler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/15dd1c12/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 8f4ee37..b87a839 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -126,7 +126,7 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 + nStopped + " stopped, " + nReady + " ready, " + 
nSUCCEED + " already succeed, " + nError
 + " error, " + nDiscarded + " discarded, " + nOthers + 
" others");
 } catch (Exception e) {
-logger.warn("Job Fetcher caught a exception " + e);
+logger.warn("Job Fetcher caught a exception ", e);
 }
 }
 }

[17/50] [abbrv] kylin git commit: KYLIN-3085 Makes sure no update on cached and shared CubeInstance

[liyang]

KYLIN-3085 Makes sure no update on cached and shared CubeInstance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03e6b8c5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03e6b8c5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03e6b8c5

Branch: refs/heads/master
Commit: 03e6b8c5d622125aca36128c6a7942f711951768
Parents: c094265
Author: Li Yang 
Authored: Sun Dec 10 10:44:12 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 14 16:53:10 2017 +0800

--
 .../persistence/RootPersistentEntity.java   |  11 +
 .../org/apache/kylin/cube/CubeInstance.java |   6 +
 .../java/org/apache/kylin/cube/CubeManager.java | 203 +--
 .../java/org/apache/kylin/cube/CubeUpdate.java  |   5 +-
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  |   2 +
 .../apache/kylin/cube/CubeManagerCacheTest.java |  22 +-
 .../kylin/cube/CubeManagerConcurrencyTest.java  | 141 +
 .../org/apache/kylin/cube/CubeManagerTest.java  | 123 +--
 .../org/apache/kylin/cube/CubeSegmentsTest.java |  38 ++--
 .../kylin/cube/project/ProjectManagerTest.java  |   8 +-
 .../metadata/cachesync/CachedCrudAssist.java|  67 --
 .../kylin/engine/mr/common/CuboidShardUtil.java |  15 +-
 .../mr/common/StatisticsDecisionUtil.java   |   6 +-
 .../engine/mr/steps/MergeDictionaryStep.java|  16 +-
 .../engine/mr/steps/MergeCuboidMapperTest.java  | 183 -
 .../apache/kylin/engine/spark/SparkCubing.java  |  31 +--
 .../kylin/provision/BuildCubeWithEngine.java|   6 +-
 .../kylin/provision/BuildCubeWithStream.java|   6 +-
 .../apache/kylin/rest/service/CubeService.java  |  39 +---
 .../apache/kylin/rest/service/JobService.java   |  13 +-
 .../kylin/source/kafka/job/MergeOffsetStep.java |  22 +-
 21 files changed, 528 insertions(+), 435 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/03e6b8c5/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
index aa35482..39c2995 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -58,6 +58,9 @@ abstract public class RootPersistentEntity implements 
AclEntity, Serializable {
 
 @JsonProperty("last_modified")
 protected long lastModified;
+
+// if cached and shared, the object MUST NOT be modified (call setXXX() 
for example)
+protected boolean isCachedAndShared = false;
 
 /**
  * Metadata model version
@@ -99,6 +102,14 @@ abstract public class RootPersistentEntity implements 
AclEntity, Serializable {
 setUuid(UUID.randomUUID().toString());
 }
 
+public boolean isCachedAndShared() {
+return isCachedAndShared;
+}
+
+public void setCachedAndShared(boolean isCachedAndShared) {
+this.isCachedAndShared = isCachedAndShared;
+}
+
 /**
  * The name as a part of the resource path used to save the entity.
  * 

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e6b8c5/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 1be7923..55e9325 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -128,6 +128,12 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 public CubeInstance() {
 }
 
+public CubeInstance latestCopyForWrite() {
+CubeManager mgr = CubeManager.getInstance(config);
+CubeInstance latest = mgr.getCube(name); // in case this object is 
out-of-date
+return mgr.copyForWrite(latest);
+}
+
 void init(KylinConfig config) {
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(config).getCubeDesc(descName);
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", descName, name);

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e6b8c5/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 

[04/50] [abbrv] kylin git commit: KYLIN-3092 Synchronize w/r operations on entity-caching managers

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index 1b248e5..28ea238 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -38,6 +38,21 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, 
getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = 
JsonAutoDetect.Visibility.NONE, setterVisibility = 
JsonAutoDetect.Visibility.NONE)
 public class TableExtDesc extends RootPersistentEntity {
 
+public static String concatRawResourcePath(String nameOnPath) {
+return ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + nameOnPath + 
".json";
+}
+
+public static String concatResourcePath(String tableIdentity, String prj) {
+return concatRawResourcePath(TableDesc.makeResourceName(tableIdentity, 
prj));
+}
+
+// returns 
+public static Pair parseResourcePath(String path) {
+return TableDesc.parseResourcePath(path);
+}
+
+// 

+
 @JsonProperty("table_name")
 private String tableIdentity;
 @JsonProperty("last_build_job_id")
@@ -65,24 +80,13 @@ public class TableExtDesc extends RootPersistentEntity {
 public TableExtDesc() {
 }
 
-public String getResourcePath() {
-return concatResourcePath(getIdentity(), project);
-}
-
-public static String concatRawResourcePath(String nameOnPath) {
-return ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + nameOnPath + 
".json";
-}
-
-public static String concatResourcePath(String tableIdentity, String prj) {
-if (prj == null)
-return ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + tableIdentity 
+ ".json";
-else
-return ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + tableIdentity 
+ "--" + prj + ".json";
+@Override
+public String resourceName() {
+return TableDesc.makeResourceName(getIdentity(), getProject());
 }
-
-// returns 
-public static Pair parseResourcePath(String path) {
-return TableDesc.parseResourcePath(path);
+
+public String getResourcePath() {
+return concatResourcePath(getIdentity(), getProject());
 }
 
 public String getProject() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
index 7b1f840..0029de2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
@@ -88,7 +88,7 @@ public class ProjectInstance extends RootPersistentEntity {
 private LinkedHashMap overrideKylinProps;
 
 public String getResourcePath() {
-return concatResourcePath(name);
+return concatResourcePath(resourceName());
 }
 
 public static String concatResourcePath(String projectName) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 4622f35..1c0254e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -70,12 +70,11 @@ public class ProjectManager {
 
 // project name => ProjrectInstance
 private CaseInsensitiveStringCache projectMap;
+private CachedCrudAssist crud;
 
-// protects concurrent operations around the projectMap, to avoid for 
example
-// writing a project in the middle of reloading it (dirty read)
+// protects concurrent operations around the cached map, to avoid for 
example
+// writing an entity in the middle of reloading it (dirty read)
 private AutoReadWriteLock prjMapLock = new AutoReadWriteLock();
-
-private CachedCrudAssist crud;
 
 private 

[41/50] [abbrv] kylin git commit: minor, add log

[liyang]

minor, add log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4eb6cd41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4eb6cd41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4eb6cd41

Branch: refs/heads/master
Commit: 4eb6cd41b31401542091cab7a8aa6f5b38ca8524
Parents: b8ddd82
Author: Hongbin Ma 
Authored: Thu Dec 28 18:51:40 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 28 18:53:55 2017 +0800

--
 .../main/java/org/apache/kylin/metadata/model/DataModelManager.java | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4eb6cd41/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
index a83f2c3..7f9fc5e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
@@ -245,6 +245,7 @@ public class DataModelManager {
 prj.getModels().add(name);
 
 desc.setOwner(owner);
+logger.info("Saving Model {} to Project {} with {} as owner", 
desc.getName(), projectName, owner);
 desc = saveDataModelDesc(desc);
 
 } finally {

[01/50] [abbrv] kylin git commit: Minor, adapt DefaultQueryTransformer's function scope to calcite supported

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 5f2eff68d -> 4d50b2697


Minor, adapt DefaultQueryTransformer's function scope to calcite supported


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ebd15d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ebd15d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ebd15d5

Branch: refs/heads/master
Commit: 3ebd15d522ee12103badd0547d9316a2e3314137
Parents: d173c7e
Author: Yifan Zhang 
Authored: Fri Dec 8 17:47:02 2017 +0800
Committer: Yifan Zhang 
Committed: Sun Dec 10 22:25:14 2017 +0800

--
 .../org/apache/kylin/query/util/DefaultQueryTransformer.java | 6 --
 .../apache/kylin/query/util/DefaultQueryTransformerTest.java | 8 
 2 files changed, 12 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ebd15d5/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java 
b/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
index b21f410..671d9db 100644
--- 
a/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
+++ 
b/query/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
@@ -43,7 +43,7 @@ public class DefaultQueryTransformer implements 
IQueryTransformer {
 private static final Pattern PTN_INTERVAL = Pattern.compile(
 "interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)",
 Pattern.CASE_INSENSITIVE);
-private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}",
+private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + SM + "(EXTRACT\\(.*?\\))" + "\\}",
 Pattern.CASE_INSENSITIVE);
 private static final Pattern PIN_SUM_OF_CAST = Pattern.compile(S0 + "SUM" 
+ S0 + "\\(" + S0 + "CAST" + S0 + "\\("
 + S0 + "([^\\s,]+)" + S0 + "AS" + SM + "DOUBLE" + S0 + "\\)" + S0 
+ "\\)", Pattern.CASE_INSENSITIVE);
@@ -76,7 +76,9 @@ public class DefaultQueryTransformer implements 
IQueryTransformer {
 }
 
 // Case {fn EXTRACT(...) }
-// Use non-greedy regrex matching to remove escape functions
+// Use non-greedy regex matching to remove escape functions
+// Notice: Only unsupported escape function need to be handled
+// Reference: 
https://calcite.apache.org/docs/reference.html#jdbc-function-escape
 while (true) {
 m = PTN_HAVING_ESCAPE_FUNCTION.matcher(sql);
 if (!m.find())

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ebd15d5/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
--
diff --git 
a/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
 
b/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
index 5a15443..8ce1017 100644
--- 
a/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
+++ 
b/query/src/test/java/org/apache/kylin/query/util/DefaultQueryTransformerTest.java
@@ -92,5 +92,13 @@ public class DefaultQueryTransformerTest {
 assertTrue("select sum(LSTG_SITE_ID), sum(price) from KYLIN_SALES 
group by LSTG_SITE_ID"
 .equalsIgnoreCase(correctSql));
 }
+
+@Test
+public void functionEscapeTransform() throws Exception {
+DefaultQueryTransformer transformer = new DefaultQueryTransformer();
 
+String fnConvertSumSql = "select {fn EXTRACT(YEAR from PART_DT)} from 
KYLIN_SALES";
+String correctSql = transformer.transform(fnConvertSumSql, "", "");
+assertTrue("select EXTRACT(YEAR from PART_DT) from 
KYLIN_SALES".equalsIgnoreCase(correctSql));
+}
 }
\ No newline at end of file

[27/50] [abbrv] kylin git commit: KYLIN-3044, KYLIN-3052, support SQL Server & Redshift as kylin data source

[liyang]

KYLIN-3044, KYLIN-3052, support SQL Server & Redshift as kylin data source


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e787d7cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e787d7cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e787d7cd

Branch: refs/heads/master
Commit: e787d7cd1b13d87ec68373893ede8c1ea534c58b
Parents: ca42b92
Author: etherge 
Authored: Mon Dec 18 14:00:26 2017 +0800
Committer: Li Yang 
Committed: Thu Dec 21 23:06:06 2017 -0600

--
 .../apache/kylin/common/KylinConfigBase.java|  22 +-
 .../main/resources/kylin-defaults.properties|   9 +
 .../org/apache/kylin/job/JoinedFlatTable.java   |  38 ++-
 .../kylin/job/constant/ExecutableConstants.java |   1 +
 pom.xml |   2 +-
 .../apache/kylin/rest/service/TableService.java |   9 +-
 source-hive/pom.xml |  25 ++
 .../apache/kylin/source/jdbc/JdbcDialect.java   |  26 ++
 .../apache/kylin/source/jdbc/JdbcExplorer.java  | 288 ---
 .../kylin/source/jdbc/JdbcHiveMRInput.java  | 132 +++--
 .../kylin/source/jdbc/JdbcTableReader.java  |  32 ++-
 .../org/apache/kylin/source/jdbc/SqlUtil.java   | 140 ++---
 .../jdbc/metadata/DefaultJdbcMetadata.java  |  76 +
 .../source/jdbc/metadata/IJdbcMetadata.java |  33 +++
 .../jdbc/metadata/JdbcMetadataFactory.java  |  35 +++
 .../source/jdbc/metadata/MySQLJdbcMetadata.java |  67 +
 .../jdbc/metadata/SQLServerJdbcMetadata.java|  61 
 .../kylin/source/jdbc/JdbcExplorerTest.java | 156 ++
 .../apache/kylin/source/jdbc/SqlUtilTest.java   |  46 +++
 .../jdbc/metadata/DefaultJdbcMetadataTest.java  | 126 
 .../jdbc/metadata/JdbcMetadataFactoryTest.java  |  35 +++
 .../jdbc/metadata/MySQLJdbcMetadataTest.java|  99 +++
 .../metadata/SQLServerJdbcMetadataTest.java |  68 +
 23 files changed, 1252 insertions(+), 274 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e787d7cd/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8572fa3..21362e9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -341,7 +341,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public String getHBaseMappingAdapter() {
 return getOptional("kylin.metadata.hbasemapping-adapter");
 }
-
+
 public boolean isCheckCopyOnWrite() {
 return 
Boolean.parseBoolean(getOptional("kylin.metadata.check-copy-on-write", 
"false"));
 }
@@ -752,23 +752,23 @@ abstract public class KylinConfigBase implements 
Serializable {
 // SOURCE.JDBC
 // 

 
-public String getJdbcConnectionUrl() {
+public String getJdbcSourceConnectionUrl() {
 return getOptional("kylin.source.jdbc.connection-url");
 }
 
-public String getJdbcDriver() {
+public String getJdbcSourceDriver() {
 return getOptional("kylin.source.jdbc.driver");
 }
 
-public String getJdbcDialect() {
+public String getJdbcSourceDialect() {
 return getOptional("kylin.source.jdbc.dialect");
 }
 
-public String getJdbcUser() {
+public String getJdbcSourceUser() {
 return getOptional("kylin.source.jdbc.user");
 }
 
-public String getJdbcPass() {
+public String getJdbcSourcePass() {
 return getOptional("kylin.source.jdbc.pass");
 }
 
@@ -776,6 +776,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.source.jdbc.sqoop-home");
 }
 
+public int getSqoopMapperNum() {
+return 
Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4"));
+}
+
+public String getFieldDelimiter() {
+return getOptional("kylin.source.jdbc.field-delimiter", "|");
+}
+
 // 

 // STORAGE.HBASE
 // 

@@ -1390,7 +1398,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public String getPerfLoggerClassName() {
 return getOptional("kylin.metrics.perflogger-class", 
"org.apache.kylin.common.metrics.perflog.PerfLogger");
 }
-
+
 public boolean isShowingGuiTraceToggle() {
 return 

[18/50] [abbrv] kylin git commit: minor, change the way that get DefaultScheduler's instance.

[liyang]

minor, change the way that get DefaultScheduler's instance.

minor, refine log when lost ZK connection.

minor, remove DefaultScheduler.destroyInstance().


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a3d11a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a3d11a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a3d11a3

Branch: refs/heads/master
Commit: 3a3d11a319024fb7eccf26eb63f7c41891d66080
Parents: 9ff03c8
Author: tttMelody <245915...@qq.com>
Authored: Wed Dec 13 14:44:52 2017 +0800
Committer: Jiatao Tao <245915...@qq.com>
Committed: Thu Dec 14 18:00:18 2017 +0800

--
 .../job/impl/threadpool/DefaultScheduler.java   | 29 +---
 .../job/impl/threadpool/BaseSchedulerTest.java  | 15 +-
 .../kylin/provision/BuildCubeWithEngine.java|  3 +-
 .../kylin/provision/BuildCubeWithStream.java|  3 +-
 4 files changed, 16 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3d11a3/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index ec5f552..327a793 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -62,11 +62,11 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 volatile boolean fetchFailed = false;
 private JobEngineConfig jobEngineConfig;
 
-private static DefaultScheduler INSTANCE = null;
+private static volatile DefaultScheduler INSTANCE = null;
 
 public DefaultScheduler() {
 if (INSTANCE != null) {
-throw new IllegalStateException("DefaultScheduler has been 
initiated.");
+throw new IllegalStateException("DefaultScheduler has been 
initiated. Use getInstance() instead.");
 }
 }
 
@@ -175,14 +175,11 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 }
 }
 
-public static DefaultScheduler getInstance() {
-return INSTANCE;
-}
-
 @Override
 public void stateChanged(CuratorFramework client, ConnectionState 
newState) {
 if ((newState == ConnectionState.SUSPENDED) || (newState == 
ConnectionState.LOST)) {
 try {
+logger.info("ZK Connection state change to " + newState + ", 
shutdown default scheduler.");
 shutdown();
 } catch (SchedulerException e) {
 throw new RuntimeException("failed to shutdown scheduler", e);
@@ -190,23 +187,11 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 }
 }
 
-public synchronized static DefaultScheduler createInstance() {
-destroyInstance();
-INSTANCE = new DefaultScheduler();
-return INSTANCE;
-}
-
-public synchronized static void destroyInstance() {
-DefaultScheduler tmp = INSTANCE;
-INSTANCE = null;
-if (tmp != null) {
-try {
-tmp.shutdown();
-} catch (SchedulerException e) {
-logger.error("error stop DefaultScheduler", e);
-throw new RuntimeException(e);
-}
+public synchronized static DefaultScheduler getInstance() {
+if (INSTANCE == null) {
+INSTANCE = new DefaultScheduler();
 }
+return INSTANCE;
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3d11a3/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index e0a542e..6786d31 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -29,7 +29,7 @@ import org.apache.kylin.job.execution.ExecutableManager;
 import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.lock.MockJobLock;
 import org.junit.After;
-import org.junit.Before;
+import org.junit.BeforeClass;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,16 +39,16 @@ public abstract class BaseSchedulerTest extends 
LocalFileMetadataTestCase {
 
 private static final Logger logger = 
LoggerFactory.getLogger(BaseSchedulerTest.class);
 
-private DefaultScheduler 

[07/50] [abbrv] kylin git commit: KYLIN-3090 Refactor to consolidate all caches and managers under KylinConfig

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
new file mode 100644
index 000..5c2ca17
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.cachesync;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+abstract public class CachedCrudAssist {
+
+private static final Logger logger = 
LoggerFactory.getLogger(CachedCrudAssist.class);
+
+final private ResourceStore store;
+final private Class entityType;
+final private String resRootPath;
+final private Serializer serializer;
+final private SingleValueCache cache;
+
+private boolean checkCopyOnWrite;
+
+public CachedCrudAssist(ResourceStore store, String resourceRootPath, 
Class entityType,
+SingleValueCache cache) {
+this.store = store;
+this.entityType = entityType;
+this.resRootPath = resourceRootPath;
+this.serializer = new JsonSerializer(entityType);
+this.cache = cache;
+
+this.checkCopyOnWrite = store.getConfig().isCheckCopyOnWrite();
+
+Preconditions.checkArgument(resRootPath.startsWith("/"));
+Preconditions.checkArgument(resRootPath.endsWith("/") == false);
+}
+
+public void setCheckOnWrite(boolean check) {
+this.checkCopyOnWrite = check;
+}
+
+private String resourcePath(String resourceName) {
+return resRootPath + "/" + resourceName + 
MetadataConstants.FILE_SURFIX;
+}
+
+public void reloadAll() throws IOException {
+logger.debug("Reloading " + entityType.getName() + " from " + 
store.getReadableResourcePath(resRootPath));
+
+cache.clear();
+
+List paths = store.collectResourceRecursively(resRootPath, 
MetadataConstants.FILE_SURFIX);
+for (String path : paths) {
+reloadQuietlyAt(path);
+}
+
+logger.debug("Loaded " + cache.size() + " " + entityType.getName() + 
"(s)");
+}
+
+public T reload(String resourceName) throws IOException {
+return reloadAt(resourcePath(resourceName));
+}
+
+public T reloadQuietly(String resourceName) {
+return reloadQuietlyAt(resourcePath(resourceName));
+}
+
+private T reloadQuietlyAt(String path) {
+try {
+return reloadAt(path);
+} catch (Exception ex) {
+logger.error("Error loading " + entityType.getName() + " at " + 
path, ex);
+return null;
+}
+}
+
+public T reloadAt(String path) throws IOException {
+T entity = store.getResource(path, entityType, serializer);
+if (entity == null) {
+logger.warn("No " + entityType.getName() + " found at " + path + 
", returning null");
+return null;
+}
+
+initEntityAfterReload(entity);
+cache.putLocal(entity.resourceName(), entity);
+return entity;
+}
+
+abstract protected void initEntityAfterReload(T entity);
+
+public T save(T entity) throws IOException {
+Preconditions.checkArgument(entity != null);
+Preconditions.checkArgument(entityType.isInstance(entity));
+
+String resName = entity.resourceName();
+if (checkCopyOnWrite) {
+if (cache.get(resName) == entity) {
+throw new IllegalStateException("Copy-on-write violation! The 

[48/50] [abbrv] kylin git commit: Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java
--
diff --cc 
core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java
index 000,904c4bd..dd18c91
mode 00,100644..100644
--- a/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java
+++ b/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java
@@@ -1,0 -1,142 +1,142 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+ */
+ 
+ package org.apache.kylin.job.metrics;
+ 
+ import org.apache.kylin.common.KylinConfig;
+ import org.apache.kylin.metrics.MetricsManager;
+ import org.apache.kylin.metrics.lib.impl.RecordEvent;
+ import org.apache.kylin.metrics.lib.impl.TimedRecordEvent;
+ import org.apache.kylin.metrics.property.JobPropertyEnum;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ 
+ public class JobMetricsFacade {
+ private static final Logger logger = 
LoggerFactory.getLogger(JobMetricsFacade.class);
+ 
+ public static void updateMetrics(JobStatisticsResult jobStats) {
+ if 
(!KylinConfig.getInstanceFromEnv().isKylinMetricsReporterForJobEnabled()) {
+ return;
+ }
+ /**
+  * report job related metrics
+  */
+ RecordEvent metricsEvent;
+ if (jobStats.throwable == null) {
+ metricsEvent = new 
TimedRecordEvent(KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectJob());
+ setJobWrapper(metricsEvent, jobStats.user, jobStats.projectName, 
jobStats.cubeName, jobStats.jobId,
+ jobStats.jobType, jobStats.cubingType);
+ setJobStats(metricsEvent, jobStats.tableSize, jobStats.cubeSize, 
jobStats.buildDuration,
+ jobStats.waitResourceTime, jobStats.perBytesTimeCost, //
+ jobStats.dColumnDistinct, jobStats.dDictBuilding, 
jobStats.dCubingInmem, jobStats.dHfileConvert);
+ } else {
+ metricsEvent = new 
TimedRecordEvent(KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectJobException());
+ setJobExceptionWrapper(metricsEvent, jobStats.user, 
jobStats.projectName, jobStats.cubeName, jobStats.jobId,
+ jobStats.jobType, jobStats.cubingType, //
+ jobStats.throwable.getClass());
+ }
+ MetricsManager.getInstance().update(metricsEvent);
+ }
+ 
+ private static void setJobWrapper(RecordEvent metricsEvent, String user, 
String projectName, String cubeName,
+ String jobId, String jobType, String cubingType) {
+ metricsEvent.put(JobPropertyEnum.USER.toString(), user);
+ metricsEvent.put(JobPropertyEnum.PROJECT.toString(), projectName);
+ metricsEvent.put(JobPropertyEnum.CUBE.toString(), cubeName);
+ metricsEvent.put(JobPropertyEnum.ID_CODE.toString(), jobId);
+ metricsEvent.put(JobPropertyEnum.TYPE.toString(), jobType);
+ metricsEvent.put(JobPropertyEnum.ALGORITHM.toString(), cubingType);
+ }
+ 
+ private static void setJobStats(RecordEvent metricsEvent, long tableSize, 
long cubeSize, long buildDuration,
+ long waitResourceTime, double perBytesTimeCost, long 
dColumnDistinct, long dDictBuilding, long dCubingInmem,
+ long dHfileConvert) {
+ metricsEvent.put(JobPropertyEnum.SOURCE_SIZE.toString(), tableSize);
+ metricsEvent.put(JobPropertyEnum.CUBE_SIZE.toString(), cubeSize);
+ metricsEvent.put(JobPropertyEnum.BUILD_DURATION.toString(), 
buildDuration);
+ metricsEvent.put(JobPropertyEnum.WAIT_RESOURCE_TIME.toString(), 
waitResourceTime);
+ metricsEvent.put(JobPropertyEnum.PER_BYTES_TIME_COST.toString(), 
perBytesTimeCost);
+ 
metricsEvent.put(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString(), 
dColumnDistinct);
+ metricsEvent.put(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString(), 
dDictBuilding);
+ 
metricsEvent.put(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString(), 
dCubingInmem);
+ 

[45/50] [abbrv] kylin git commit: Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --cc 
server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 9afec83,17f6b58..9789b70
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@@ -102,11 -98,8 +102,12 @@@ import org.apache.kylin.rest.request.Pr
  import org.apache.kylin.rest.request.SQLRequest;
  import org.apache.kylin.rest.response.SQLResponse;
  import org.apache.kylin.rest.util.AclEvaluate;
 +import org.apache.kylin.rest.util.AclPermissionUtil;
+ import org.apache.kylin.rest.util.QueryRequestLimits;
  import org.apache.kylin.rest.util.TableauInterceptor;
 +import org.apache.kylin.shaded.htrace.org.apache.htrace.Sampler;
 +import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
 +import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
  import org.apache.kylin.storage.hybrid.HybridInstance;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
@@@ -403,163 -403,121 +404,176 @@@ public class QueryService extends Basic
  
  final QueryContext queryContext = QueryContext.current();
  
 +TraceScope scope = null;
- if (KylinConfig.getInstanceFromEnv().isHtraceTracingEveryQuery() || 
BackdoorToggles.getHtraceEnabled()) {
++if (kylinConfig.isHtraceTracingEveryQuery() || 
BackdoorToggles.getHtraceEnabled()) {
 +logger.info("Current query is under tracing");
 +HtraceInit.init();
 +scope = Trace.startSpan("query life cycle for " + 
queryContext.getQueryId(), Sampler.ALWAYS);
 +}
 +String traceUrl = getTraceUrl(scope);
 +
  try (SetThreadName ignored = new SetThreadName("Query %s", 
queryContext.getQueryId())) {
- 
++long startTime = System.currentTimeMillis();
++
++SQLResponse sqlResponse = null;
  String sql = sqlRequest.getSql();
  String project = sqlRequest.getProject();
++boolean isQueryCacheEnabled = isQueryCacheEnabled(kylinConfig);
  logger.info("Using project: " + project);
  logger.info("The original query:  " + sql);
  
 -final boolean isSelect = QueryUtil.isSelectStatement(sql);
 -final boolean isPushDownUpdateEnabled = 
kylinConfig.isPushDownEnabled()
 -&& kylinConfig.isPushDownUpdateEnabled();
 -final int maxConcurrentQuery = 
projectInstance.getConfig().getQueryConcurrentRunningThresholdForProject();
 +sql = QueryUtil.removeCommentInSql(sql);
  
 -if (!isSelect && !isPushDownUpdateEnabled) {
 -logger.debug("Directly return exception as the sql is 
unsupported, and query pushdown is disabled");
 -throw new BadRequestException(msg.getNOT_SUPPORTED_SQL());
 -}
 -
 -SQLResponse sqlResponse = null;
 +Pair result = 
TempStatementUtil.handleTempStatement(sql, kylinConfig);
- 
- boolean isTempStatement = result.getFirst();
++boolean isCreateTempStatement = result.getFirst();
 +sql = result.getSecond();
 +sqlRequest.setSql(sql);
  
- final boolean isSelect = QueryUtil.isSelectStatement(sql);
- 
- long startTime = System.currentTimeMillis();
- 
- SQLResponse sqlResponse = null;
- boolean queryCacheEnabled = 
checkCondition(kylinConfig.isQueryCacheEnabled(),
- "query cache disabled in KylinConfig") && //
- checkCondition(!BackdoorToggles.getDisableCache(), "query 
cache disabled in BackdoorToggles");
- 
- if (queryCacheEnabled) {
 -try {
 -// Check project level query request concurrency limitation 
per query server
 -if 
(!QueryRequestLimits.openQueryRequest(projectInstance.getName(), 
maxConcurrentQuery)) {
 -logger.warn(
 -"Directly return exception as too many concurrent 
query requests for project:" + project);
 -throw new 
BadRequestException(msg.getQUERY_TOO_MANY_RUNNING());
++// try some cheap executions
++if (sqlResponse == null && isQueryInspect) {
++sqlResponse = new SQLResponse(null, null, 0, false, 
sqlRequest.getSql());
++}
++
++if (sqlResponse == null && isCreateTempStatement) {
++sqlResponse = new SQLResponse(null, null, 0, false, null);
++}
++
++if (sqlResponse == null && isQueryCacheEnabled) {
 +sqlResponse = searchQueryInCache(sqlRequest);
 +Trace.addTimelineAnnotation("query cache searched");
-