kylin git commit: KYLIN-2202 code review

2016-11-23 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master af3e4fece -> 827205f17


KYLIN-2202 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/827205f1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/827205f1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/827205f1

Branch: refs/heads/master
Commit: 827205f17b61d1e1ac9d6af580e04583c1079003
Parents: af3e4fe
Author: Li Yang 
Authored: Thu Nov 24 14:52:33 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 24 15:18:19 2016 +0800

--
 .../validation/rule/DictionaryRuleTest.java |   1 -
 .../apache/kylin/dict/DictionaryGenerator.java  |   7 +-
 .../apache/kylin/dict/DictionaryManager.java|   4 -
 .../dict/NumberDictionaryForestBuilder.java |   4 -
 .../kylin/dict/TrieDictionaryForestBuilder.java |   3 +-
 .../apache/kylin/engine/mr/DFSFileTable.java|  35 ---
 .../engine/mr/DFSSingleFileTableReader.java | 218 ---
 .../apache/kylin/engine/mr/SortedColumn.java| 100 -
 .../kylin/engine/mr/SortedColumnDFSFile.java| 125 +++
 .../engine/mr/SortedColumnDFSFileReader.java| 136 
 .../kylin/engine/mr/SortedColumnReader.java | 136 
 .../engine/mr/steps/CreateDictionaryJob.java|   6 +-
 .../mr/steps/FactDistinctColumnsReducer.java|   3 -
 .../mr/steps/FactDistinctHiveColumnsMapper.java |  12 +-
 .../engine/mr/steps/SelfDefineSortableKey.java  |  25 ++-
 .../apache/kylin/engine/mr/steps/TypeFlag.java  |  28 ---
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   4 +-
 .../kylin/engine/mr/SortedColumnReaderTest.java | 167 +++---
 .../mr/steps/NumberDictionaryForestTest.java|   1 +
 .../mr/steps/SelfDefineSortableKeyTest.java |  93 
 .../kylin/source/hive/HiveTableReader.java  |   3 -
 21 files changed, 412 insertions(+), 699 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/827205f1/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
 
b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index 8bd4c88..9b37507 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -34,7 +34,6 @@ import org.apache.kylin.cube.model.validation.ValidateContext;
 import org.apache.kylin.dict.GlobalDictionaryBuilder;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 /**

http://git-wip-us.apache.org/repos/asf/kylin/blob/827205f1/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 20a57ba..810a392 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -135,8 +135,7 @@ public class DictionaryGenerator {
 private static class StringDictBuilder implements IDictionaryBuilder {
 @Override
 public Dictionary build(DictionaryInfo dictInfo, 
IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, 
ArrayList returnSamples) throws IOException {
-int maxTrieSizeInMB = 
TrieDictionaryForestBuilder.getMaxTrieSizeInMB();
-TrieDictionaryForestBuilder builder = new 
TrieDictionaryForestBuilder(new StringBytesConverter(), baseId, 
maxTrieSizeInMB);
+TrieDictionaryForestBuilder builder = new 
TrieDictionaryForestBuilder(new StringBytesConverter(), baseId);
 String value;
 while (valueEnumerator.moveNext()) {
 value = valueEnumerator.current();
@@ -153,9 +152,7 @@ public class DictionaryGenerator {
 private static class NumberDictBuilder implements IDictionaryBuilder {
 @Override
 public Dictionary build(DictionaryInfo dictInfo, 
IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, 
ArrayList returnSamples) throws IOException {
-
-int maxTrieSizeInMB = 
TrieDictionaryForestBuilder.getMaxTrieSizeInMB();
-NumberDictionaryForestBuilder builder = new 
NumberDictionaryForestBuilder(baseId, maxTrieSizeInMB);
+NumberDictionaryForestBuilder builder = new

kylin git commit: KYLIN-2229 update source-assembly.xml

2016-11-23 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/master 873f903f3 -> af3e4fece


KYLIN-2229 update source-assembly.xml


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af3e4fec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af3e4fec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af3e4fec

Branch: refs/heads/master
Commit: af3e4fece65b7f7add39bb016ba03e0e7774ab3e
Parents: 873f903
Author: shaofengshi 
Authored: Thu Nov 24 09:44:22 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 24 13:38:25 2016 +0800

--
 assembly/src/main/config/assemblies/source-assembly.xml | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/af3e4fec/assembly/src/main/config/assemblies/source-assembly.xml
--
diff --git a/assembly/src/main/config/assemblies/source-assembly.xml 
b/assembly/src/main/config/assemblies/source-assembly.xml
index 23a215b..fad45aa 100644
--- a/assembly/src/main/config/assemblies/source-assembly.xml
+++ b/assembly/src/main/config/assemblies/source-assembly.xml
@@ -92,12 +92,14 @@ limitations under the License.
 
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?dist(/.*)?]
 
 
-
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?build(/.*)?]
-
 
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?lib(/.*)?]
 
 
%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?docs(/.*)?]
 
+
+
+
%regex[(?!((?!${project.build.directory}/)))?engine-spark(/.*)?]
+

kylin git commit: minor, refactor AggregationGroup.calculateCuboidCombination()

2016-11-23 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 798f03ed2 -> 873f903f3


minor, refactor AggregationGroup.calculateCuboidCombination()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/873f903f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/873f903f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/873f903f

Branch: refs/heads/master
Commit: 873f903f3d95afa2cec5e1d6d3027f1b62f3ec54
Parents: 798f03e
Author: Li Yang 
Authored: Thu Nov 24 13:00:42 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 24 13:00:42 2016 +0800

--
 .../common/util/LocalFileMetadataTestCase.java  |  4 +-
 .../kylin/cube/model/AggregationGroup.java  | 88 ++
 .../validation/rule/AggregationGroupRule.java   | 12 +--
 .../kylin/cube/AggregationGroupRuleTest.java|  3 -
 .../org/apache/kylin/cube/CubeDescTest.java | 97 +---
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  5 +-
 6 files changed, 141 insertions(+), 68 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/873f903f/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index d351041..26e1e86 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -46,7 +46,7 @@ public class LocalFileMetadataTestCase extends 
AbstractKylinTestCase {
 public static void staticCreateTestMetadata(String testDataFolder) {
 KylinConfig.destroyInstance();
 
-String tempTestMetadataUrl = "../examples/test_metadata";
+String tempTestMetadataUrl = LOCALMETA_TEMP_DATA;
 try {
 FileUtils.deleteDirectory(new File(tempTestMetadataUrl));
 FileUtils.copyDirectory(new File(testDataFolder), new 
File(tempTestMetadataUrl));
@@ -61,7 +61,7 @@ public class LocalFileMetadataTestCase extends 
AbstractKylinTestCase {
 }
 
 public static void cleanAfterClass() {
-String tempTestMetadataUrl = "../examples/test_metadata";
+String tempTestMetadataUrl = LOCALMETA_TEMP_DATA;
 try {
 FileUtils.deleteDirectory(new File(tempTestMetadataUrl));
 } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/873f903f/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index ab63c00..12c0574 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -19,9 +19,12 @@
 package org.apache.kylin.cube.model;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 
-import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -34,9 +37,9 @@ import com.google.common.collect.Lists;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class AggregationGroup {
 public static class HierarchyMask {
-public long fullMask;// 0111
-public long[] allMasks;// 0100,0110,0111
-public long[] dims;// 0100,0010,0001
+public long fullMask; // 0111
+public long[] allMasks; // 0100,0110,0111
+public long[] dims; // 0100,0010,0001
 }
 
 @JsonProperty("includes")
@@ -64,7 +67,7 @@ public class AggregationGroup {
 throw new IllegalStateException("AggregationGroup incomplete");
 }
 
-columnNamesToUpperCase();
+normalizeColumnNames();
 
 buildPartialCubeFullMask(rowKeyDesc);
 buildMandatoryColumnMask(rowKeyDesc);
@@ -76,19 +79,41 @@ public class AggregationGroup {
 
 }
 
-private void columnNamesToUpperCase() {
-StringUtil.toUpperCaseArray(includes, includes);
-StringUtil.toUpperCaseArray(selectRule.mandatory_dims, 
selectRule.mandatory_dims);
-if (selectRule.hierarchy_dims != null) {
-for (String[] cols :

kylin git commit: update upgrade guide for 1.6.0

2016-11-23 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/document 1c82143d2 -> 7dfcc595b


update upgrade guide for 1.6.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7dfcc595
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7dfcc595
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7dfcc595

Branch: refs/heads/document
Commit: 7dfcc595b7bd58ae4f71df6ea6b9c2227d992cb3
Parents: 1c82143
Author: shaofengshi 
Authored: Thu Nov 24 12:28:58 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 24 12:28:58 2016 +0800

--
 website/_docs15/howto/howto_upgrade.md |  12 +++
 website/_docs16/howto/howto_upgrade.md | 127 
 website/download/index.cn.md   |  10 +--
 website/download/index.md  |  10 +--
 4 files changed, 40 insertions(+), 119 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7dfcc595/website/_docs15/howto/howto_upgrade.md
--
diff --git a/website/_docs15/howto/howto_upgrade.md 
b/website/_docs15/howto/howto_upgrade.md
index 637158e..6bbda13 100644
--- a/website/_docs15/howto/howto_upgrade.md
+++ b/website/_docs15/howto/howto_upgrade.md
@@ -6,6 +6,18 @@ permalink: /docs15/howto/howto_upgrade.html
 since: v1.5.1
 ---
 
+Running as a Hadoop client, Apache Kylin's metadata and Cube data are 
persistended in Hadoop (HBase and HDFS), so the upgrade is relatively easy and 
user doesn't need worry about data loss. The upgrade can be performed in the 
following steps:
+
+* Download the new Apache Kylin binary package for your Hadoop version from 
Kylin's download page;
+* Uncompress the new version Kylin package to a new folder, e.g, 
/usr/local/kylin/apache-kylin-1.6.0/ (directly overwrite old instance is not 
recommended);
+* Copy the configuration files (`$KYLIN_HOME/conf/*`) from old instance (e.g 
/usr/local/kylin/apache-kylin-1.5.4/) to the new instance's `conf` folder if 
you have customized configurations; It is recommended to do a compare and merge 
since there might be new parameters introduced. If you have modified tomcat 
configuration ($KYLIN_HOME/tomcat/conf/), also remember to do the same.
+* Stop the current Kylin instance with `./bin/kylin.sh stop`;
+* Set the `KYLIN_HOME` env variable to the new instance's installation folder. 
If you have set `KYLIN_HOME` in `~/.bash_profile` or other scripts, remember to 
update them as well.
+* Start the new Kylin instance with `$KYLIN_HOME/bin/kylin start`; After be 
started, login Kylin web to check whether your cubes can be loaded correctly.
+* [Upgrade coprocessor](howto_update_coprocessor.html) to ensure the HBase 
region servers use the latest Kylin coprocessor.
+* Verify your SQL queries can be performed successfully.
+
+Below are versions specific guides:
 
 ## Upgrade from 1.5.2 to v1.5.3
 Kylin v1.5.3 metadata is compitible with v1.5.2, your cubes don't need 
rebuilt, as usual, some actions need to be performed:

http://git-wip-us.apache.org/repos/asf/kylin/blob/7dfcc595/website/_docs16/howto/howto_upgrade.md
--
diff --git a/website/_docs16/howto/howto_upgrade.md 
b/website/_docs16/howto/howto_upgrade.md
index 47f0414..ed53116 100644
--- a/website/_docs16/howto/howto_upgrade.md
+++ b/website/_docs16/howto/howto_upgrade.md
@@ -6,6 +6,24 @@ permalink: /docs16/howto/howto_upgrade.html
 since: v1.5.1
 ---
 
+Running as a Hadoop client, Apache Kylin's metadata and Cube data are 
persistended in Hadoop (HBase and HDFS), so the upgrade is relatively easy and 
user doesn't need worry about data loss. The upgrade can be performed in the 
following steps:
+
+* Download the new Apache Kylin binary package for your Hadoop version from 
Kylin's download page;
+* Uncompress the new version Kylin package to a new folder, e.g, 
/usr/local/kylin/apache-kylin-1.6.0/ (directly overwrite old instance is not 
recommended);
+* Copy the configuration files (`$KYLIN_HOME/conf/*`) from old instance (e.g 
/usr/local/kylin/apache-kylin-1.5.4/) to the new instance's `conf` folder if 
you have customized configurations; It is recommended to do a compare and merge 
since there might be new parameters introduced. If you have modified tomcat 
configuration ($KYLIN_HOME/tomcat/conf/), also remember to do the same.
+* Stop the current Kylin instance with `./bin/kylin.sh stop`;
+* Set the `KYLIN_HOME` env variable to the new instance's installation folder. 
If you have set `KYLIN_HOME` in `~/.bash_profile` or other scripts, remember to 
update them as well.
+* Start the new Kylin instance with `$KYLIN_HOME/bin/kylin start`; After be 
started, login Kylin web to check whether your cubes can be loaded correctly.
+* [Upgrade

kylin git commit: minor, cleanup kylin_rest_address in kylin.sh

2016-11-23 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 83adb8eb8 -> 53b5a6d8e


minor, cleanup kylin_rest_address in kylin.sh


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/53b5a6d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/53b5a6d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/53b5a6d8

Branch: refs/heads/master
Commit: 53b5a6d8e2fa854cbf09ea66fc39bdfbd961950f
Parents: 83adb8e
Author: Yang Li 
Authored: Wed Nov 23 22:28:38 2016 +0800
Committer: Yang Li 
Committed: Wed Nov 23 22:28:38 2016 +0800

--
 build/bin/kylin.sh| 10 +-
 .../apache/kylin/common/BackwardCompatibilityConfig.java  |  2 ++
 2 files changed, 3 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/53b5a6d8/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index d3aa4cb..fca1e5c 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -86,14 +86,7 @@ then
 #additionally add tomcat libs to HBASE_CLASSPATH_PREFIX
 export 
HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:${HBASE_CLASSPATH_PREFIX}
 
-if [ -z "$KYLIN_REST_ADDRESS" ]
-then
-kylin_rest_address=`hostname -f`":"`grep "/dev/null 2>&1; nc_result=$?
 if [ $nc_result -eq 0 ]; then
@@ -119,7 +112,6 @@ then
 -Dkylin.hive.dependency=${hive_dependency} \
 -Dkylin.hbase.dependency=${hbase_dependency} \
 -Dkylin.kafka.dependency=${kafka_dependency} \
--Dkylin.rest.address=${kylin_rest_address} \
 -Dspring.profiles.active=${spring_profile} \
 org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar  
org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.out 
2>&1 & echo $! > ${KYLIN_HOME}/pid &
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/53b5a6d8/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
 
b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
index 664d9bb..21df932 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
@@ -185,6 +185,8 @@ public class BackwardCompatibilityConfig {
 // exclude webapp/app/components
 if (f.getName().equals("components") && 
f.getParentFile().getName().equals("app"))
 return false;
+else if (f.getName().equals("node_modules") && 
f.getParentFile().getName().equals("webapp"))
+return false;
 else if (f.getName().equals("target"))
 return false;
 else

kylin git commit: KYLIN-227

2016-11-23 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-227 [created] f8aa797ac


KYLIN-227


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8aa797a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8aa797a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8aa797a

Branch: refs/heads/KYLIN-227
Commit: f8aa797ac3c1e6c5d3f4bf8343b14dcc158c387f
Parents: 9f21d2a
Author: shaofengshi 
Authored: Fri Nov 11 16:11:55 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 23 18:40:25 2016 +0800

--
 .../kylin/job/constant/JobStatusEnum.java   |  2 +-
 .../kylin/job/constant/JobStepStatusEnum.java   |  2 +-
 .../kylin/job/execution/AbstractExecutable.java |  5 +++
 .../job/execution/DefaultChainedExecutable.java |  6 
 .../kylin/job/execution/ExecutableManager.java  | 11 +-
 .../kylin/job/execution/ExecutableState.java|  7 
 .../job/impl/threadpool/DefaultScheduler.java   |  6 ++--
 .../apache/kylin/job/ExecutableManagerTest.java |  2 +-
 .../test_case_data/sandbox/kylin.properties |  6 ++--
 .../kylin/rest/controller/JobController.java| 22 
 .../apache/kylin/rest/service/CubeService.java  |  2 +-
 .../apache/kylin/rest/service/JobService.java   | 15 
 webapp/app/js/controllers/job.js| 38 +++-
 webapp/app/js/model/jobConfig.js|  1 +
 webapp/app/js/services/jobs.js  |  3 +-
 webapp/app/partials/jobs/jobList.html   | 10 +-
 16 files changed, 125 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f8aa797a/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
index a4ef564..4c6ac97 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
@@ -20,7 +20,7 @@ package org.apache.kylin.job.constant;
 
 public enum JobStatusEnum {
 
-NEW(0), PENDING(1), RUNNING(2), FINISHED(4), ERROR(8), DISCARDED(16);
+NEW(0), PENDING(1), RUNNING(2), FINISHED(4), ERROR(8), DISCARDED(16), 
STOPPED(32);
 
 private final int code;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8aa797a/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
index 08ee79a..08cd138 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java
@@ -19,7 +19,7 @@
 package org.apache.kylin.job.constant;
 
 public enum JobStepStatusEnum {
-NEW(0), PENDING(1), RUNNING(2), FINISHED(4), ERROR(8), DISCARDED(16), 
WAITING(32), KILLED(64);
+NEW(0), PENDING(1), RUNNING(2), FINISHED(4), ERROR(8), DISCARDED(16), 
WAITING(32), KILLED(64), STOPPED(128);
 
 private final int code;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8aa797a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 80a92de..551241b 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -380,6 +380,11 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 return status == ExecutableState.DISCARDED;
 }
 
+protected final boolean isPaused() {
+final ExecutableState status = getOutput().getState();
+return status == ExecutableState.STOPPED;
+}
+
 protected boolean needRetry() {
 return this.retry <= config.getJobRetry();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f8aa797a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index fccab30..253072e 100644
---

svn commit: r1770943 - in /kylin/site: ./ development/

2016-11-23 Thread lidong 
Author: lidong
Date: Wed Nov 23 10:22:37 2016
New Revision: 1770943

URL: http://svn.apache.org/viewvc?rev=1770943=rev
Log:
add coding and naming convention

Added:
kylin/site/development/coding_naming_convention.html
Modified:
kylin/site/development/about_temp_files.html
kylin/site/development/dev_env.html
kylin/site/development/howto_contribute.html
kylin/site/development/howto_docs.html
kylin/site/development/howto_hbase_branches.html
kylin/site/development/howto_package.html
kylin/site/development/howto_release.html
kylin/site/development/howto_test.html
kylin/site/development/index.html
kylin/site/development/new_metadata.html
kylin/site/development/plugin_arch.html
kylin/site/development/web_tech.html
kylin/site/feed.xml

Modified: kylin/site/development/about_temp_files.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/about_temp_files.html?rev=1770943=1770942=1770943=diff
==
--- kylin/site/development/about_temp_files.html (original)
+++ kylin/site/development/about_temp_files.html Wed Nov 23 10:22:37 2016
@@ -211,6 +211,8 @@
 
   
 
+  
+
   Development Quick Guide  
   
 
@@ -228,6 +230,8 @@
 
   
 
+  
+
   Setup Development Env  
   
 
@@ -245,6 +249,25 @@
 
   
 
+  Coding and Naming 
Convention  
+  
+
+
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
   
 
   
@@ -276,6 +299,8 @@
 
   
 
+  
+
   How to Contribute
  
   
 
@@ -297,6 +322,8 @@
 
   
 
+  
+
   How to Write Document  
   
 
@@ -322,6 +349,8 @@
 
   
 
+  
+
   How to Build Binary Package  
   
 
@@ -345,6 +374,8 @@
 
   
 
+  
+
   How to Maintain HBase 
Branches  
   
 
@@ -372,6 +403,8 @@
 
   
 
+  
+
   How to Make Release  
   
 
@@ -407,6 +440,8 @@
 
   
 
+  
+
   Plugin Architecture  
   
 
@@ -440,6 +475,8 @@
 
   
 
+  
+
   New Metadata Model  
   
 
@@ -456,6 +493,8 @@
   
 
   
+
+  
 
   
 

Added: kylin/site/development/coding_naming_convention.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/coding_naming_convention.html?rev=1770943=auto
==
--- kylin/site/development/coding_naming_convention.html (added)
+++ kylin/site/development/coding_naming_convention.html Wed Nov 23 10:22:37 
2016
@@ -0,0 +1,665 @@
+
+
+
+
+   
+
+
+  
+  
+  
+
+  Apache Kylin | Coding and Naming Convention
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+  
+  
+
+  http://kylin.apache.org/development/coding_naming_convention.html;>
+  http://kylin.apache.org/feed.xml; />
+
+
+
+  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new 
Date();a=s.createElement(o),
+  
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+  //oringal tracker for kylin.io
+  ga('create', 'UA-55534813-1', 'auto');
+  //new tracker for kylin.apache.org
+  ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'});
+
+  ga('send', 'pageview');
+  ga('toplevel.send', 'pageview');
+
+
+
+
+ 
+ 
+
+
+   
+   
+
+
+  
+  
+  Apache Kylin™ Extreme OLAP Engine for Big 
Data 
+
+  
+  
+
+  
+  
+  
+
+
+
+  
+Toggle navigation
+
+
+
+  
+ 
+
+
+
+
+  
+ Home
+  Docs
+  Download
+  Community
+  Development
+  Blog
+  中文版  
+  https://twitter.com/apachekylin; target="_blank" 
class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" >
+  https://github.com/apache/kylin; target="_blank" 
class="fa fa-github-alt fa-lg" title="Github: apache/kylin" >  
+  https://www.facebook.com/kylinio; target="_blank" 
class="fa fa-facebook fa-lg" title="Facebook: kylin.io" >   
+
+
+  
+
+ 
+
+   
+   
+   
+   
+
+
+
+
+Development Guide
+
+   
+
+
+
+
+
+
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Development Quick Guide  
+  
+
+
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  Setup Development Env  
+  
+
+
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  Coding and Naming 
Convention  
+  
+
+
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  How to Test  
+  
+
+
+   
+

kylin git commit: add coding and naming convention

2016-11-23 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/document e18fd358f -> 1c82143d2


add coding and naming convention


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c82143d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c82143d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c82143d

Branch: refs/heads/document
Commit: 1c82143d2040ce6f16060c1bc252dc78fe69d763
Parents: e18fd35
Author: Li Yang 
Authored: Wed Nov 23 18:20:28 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 23 18:20:28 2016 +0800

--
 website/_data/development.yml|  1 +
 website/_dev/coding_naming_convention.md | 61 +++
 2 files changed, 62 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c82143d/website/_data/development.yml
--
diff --git a/website/_data/development.yml b/website/_data/development.yml
index 8acd463..827fb57 100644
--- a/website/_data/development.yml
+++ b/website/_data/development.yml
@@ -20,6 +20,7 @@
   dev:
   - index
   - dev_env
+  - coding_naming_convention
   - howto_test
   - howto_contribute
   - howto_docs

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c82143d/website/_dev/coding_naming_convention.md
--
diff --git a/website/_dev/coding_naming_convention.md 
b/website/_dev/coding_naming_convention.md
new file mode 100644
index 000..ffcd77d
--- /dev/null
+++ b/website/_dev/coding_naming_convention.md
@@ -0,0 +1,61 @@
+---
+layout: dev
+title:  Coding and Naming Convention
+categories: development
+permalink: /development/coding_naming_convention.html
+---
+
+## Coding Convention
+
+Coding convention is very important for teamwork. Not only it keeps code neat 
and tidy, it saves a lot of work too. Different coding convention (and auto 
formatter) will cause unnecessary code changes that requires more effort at 
code review and code merge.
+
+For Java code, we use Eclipse default formatter setting, with one change that 
to allow long lines.
+
+- For Eclipse developers, no manual setting is required. Code formatter 
configurations `.settings/org.eclipse.jdt.core.prefs` is on git repo. Your IDE 
should be auto configured when the projects are imported.
+- For intellij IDEA developers, you need to install "Eclipse Code Formatter" 
and load the Eclipse formatter settings into your IDE manually. See [Setup 
Development Env](dev_env.html) for details.
+- We have *checkstyle plugin* enabled in maven to enforce some convention 
checks.
+
+For JavaScript, XML, and other code, please use space for indent. And as a 
general rule, keep your code format consistent with existing lines. No other 
enforcement at the moment.
+
+
+
+## Configuration Naming Convention
+
+For Kylin configuration names (those in `kylin.properties`)
+
+- The convention is `dot.separated.namespace.config-name-separated-by-dash`, 
all chars in lower case.
+- Rationale: The dot separated prefix is for namespace, like java packages. 
The last level is like class name, but in lower case and separated by dash. The 
result is consistent with common hadoop config names, i.e. 
`dfs.namenode.servicerpc-bind-host`.
+- Good examples: `kylin.metadata.url`; `kylin.snapshot.max-mb`
+- Bad exmaples: `kylin.cube.customEncodingFactories`, should be 
`kylin.cube.custom-encoding-factories`
+- Namespaces (or packages) of the names should map from the Java project and 
package where the configuration is used. Below is a list of current namespaces.
+  - kylin.env
+  - kylin.metadata
+  - kylin.snapshot
+  - kylin.dictionary
+  - kylin.cube
+  - kylin.job
+  - kylin.engine
+  - kylin.engine.mr
+  - kylin.engine.spark
+  - kylin.source
+  - kylin.source.hive
+  - kylin.source.kafka
+  - kylin.storage
+  - kylin.storage.hbase
+  - kylin.query
+  - kylin.security
+  - kylin.server
+  - kylin.web
+
+
+
+## Configuration File Naming Convention
+
+For configuration files like logging config, spring config, mapreduce job 
config etc.
+
+- The convention is `words-separated-by-dash.ext`, all chars in lower case.
+- Rationale: Be consistent with hadoop config file names, i.e. hdfs-site.xml
+- Good example: `kylin-server-log4j.properties`
+- Bad example: `kylin_hive_conf.xml`, should be `kylin-hive-conf.xml`
+
+

[1/2] kylin git commit: minor, fix kylin.sh version

2016-11-23 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 3e4e82f06 -> 83adb8eb8


minor, fix kylin.sh version


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83adb8eb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83adb8eb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83adb8eb

Branch: refs/heads/master
Commit: 83adb8eb8700d388aa042d833e7fea9746282624
Parents: b8e764d
Author: Hongbin Ma 
Authored: Wed Nov 23 18:14:40 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 23 18:14:45 2016 +0800

--
 build/bin/kylin.sh | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83adb8eb/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index b0627ef..d3aa4cb 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -151,6 +151,7 @@ then
 
 elif [ "$1" = "version" ]
 then
+retrieveDependency
 exec hbase 
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties 
org.apache.kylin.common.KylinVersion
 exit 0

[2/2] kylin git commit: KYLIN-2227 rename kylin-log4j.properties to kylin-tools-log4j.properties and move it to global conf folder

2016-11-23 Thread mahongbin 
KYLIN-2227 rename kylin-log4j.properties to kylin-tools-log4j.properties and 
move it to global conf folder


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8e764d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8e764d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8e764d4

Branch: refs/heads/master
Commit: b8e764d434ab12b9fb7b902ac8dea0d6b792dec5
Parents: 3e4e82f
Author: Hongbin Ma 
Authored: Wed Nov 23 17:46:04 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 23 18:14:45 2016 +0800

--
 build/bin/kylin.sh  |  4 +--
 build/conf/kylin-tools-log4j.properties | 36 
 .../src/main/resources/kylin-log4j.properties   | 28 ---
 .../gridtable/AggregationCacheSpillTest.java|  1 -
 kylin-it/pom.xml|  6 ++--
 pom.xml |  2 +-
 .../java/org/apache/kylin/rest/DebugTomcat.java |  2 +-
 7 files changed, 43 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b8e764d4/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index fff6227..b0627ef 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -151,7 +151,7 @@ then
 
 elif [ "$1" = "version" ]
 then
-exec hbase -Dlog4j.configuration=kylin-log4j.properties 
org.apache.kylin.common.KylinVersion
+exec hbase 
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties 
org.apache.kylin.common.KylinVersion
 exit 0
 
 elif [ "$1" = "diag" ]
@@ -170,7 +170,7 @@ then
 then source ${dir}/setenv-tool.sh
 fi
 
-exec hbase ${KYLIN_EXTRA_START_OPTS} 
-Dkylin.hive.dependency=${hive_dependency} 
-Dkylin.hbase.dependency=${hbase_dependency} 
-Dlog4j.configuration=kylin-log4j.properties "$@"
+exec hbase ${KYLIN_EXTRA_START_OPTS} 
-Dkylin.hive.dependency=${hive_dependency} 
-Dkylin.hbase.dependency=${hbase_dependency} 
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties "$@"
 
 else
 quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop'"

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8e764d4/build/conf/kylin-tools-log4j.properties
--
diff --git a/build/conf/kylin-tools-log4j.properties 
b/build/conf/kylin-tools-log4j.properties
new file mode 100644
index 000..0ba7d2e
--- /dev/null
+++ b/build/conf/kylin-tools-log4j.properties
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#  
+# http://www.apache.org/licenses/LICENSE-2.0
+#  
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+# the kylin-tools-log4j.properties is mainly for configuring log properties on 
kylin tools, including:
+#   1. tools launched by kylin.sh script, e.g. DeployCoprocessorCLI
+#   2. DebugTomcat
+#   3. others
+# 
+# It's called kylin-tools-log4j.properties so that it won't distract users 
from the other more important log4j config file: kylin-server-log4j.properties  
+# enable this by -Dlog4j.configuration=kylin-tools-log4j.properties
+
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: 
%m%n
+
+#log4j.logger.org.apache.hadoop=ERROR
+log4j.logger.org.apache.kylin=DEBUG
+log4j.logger.org.springframework=WARN

http://git-wip-us.apache.org/repos/asf/kylin/blob/b8e764d4/core-common/src/main/resources/kylin-log4j.properties
--
diff --git a/core-common/src/main/resources/kylin-log4j.properties 
b/core-common/src/main/resources/kylin-log4j.properties
deleted file mode 100644
index e7201b3..000
--- a/core-common/src/main/resources/kylin-log4j.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file

[7/9] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-23 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
index b3facd2..fbb93db 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
@@ -1,63 +1,63 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.kv;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dimension.DictionaryDimEnc;
-import org.apache.kylin.dimension.DimensionEncoding;
-import org.apache.kylin.dimension.IDimensionEncodingMap;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * Read/Write column values from/into bytes
- *
- * @author yangli9
- */
-public class RowKeyColumnIO {
-
-//private static final Logger logger = 
LoggerFactory.getLogger(RowKeyColumnIO.class);
-
-private final IDimensionEncodingMap dimEncMap;
-
-public RowKeyColumnIO(IDimensionEncodingMap dimEncMap) {
-this.dimEncMap = dimEncMap;
-}
-
-public int getColumnLength(TblColRef col) {
-return dimEncMap.get(col).getLengthOfEncoding();
-}
-
-public Dictionary getDictionary(TblColRef col) {
-return dimEncMap.getDictionary(col);
-}
-
-public void writeColumn(TblColRef col, String value, int roundingFlag, 
byte defaultValue, byte[] output, int outputOffset) {
-DimensionEncoding dimEnc = dimEncMap.get(col);
-if (dimEnc instanceof DictionaryDimEnc)
-dimEnc = ((DictionaryDimEnc) dimEnc).copy(roundingFlag, 
defaultValue);
-
-dimEnc.encode(value, output, outputOffset);
-}
-
-public String readColumnString(TblColRef col, byte[] bytes, int offset, 
int length) {
-DimensionEncoding dimEnc = dimEncMap.get(col);
-return dimEnc.decode(bytes, offset, length);
-}
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.kv;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.dimension.IDimensionEncodingMap;
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ * Read/Write column values from/into bytes
+ *
+ * @author yangli9
+ */
+public class RowKeyColumnIO {
+
+//private static final Logger logger = 
LoggerFactory.getLogger(RowKeyColumnIO.class);
+
+private final IDimensionEncodingMap dimEncMap;
+
+public RowKeyColumnIO(IDimensionEncodingMap dimEncMap) {
+this.dimEncMap = dimEncMap;
+}
+
+public int getColumnLength(TblColRef col) {
+return dimEncMap.get(col).getLengthOfEncoding();
+}
+
+public Dictionary getDictionary(TblColRef col) {
+return dimEncMap.getDictionary(col);
+}
+
+public void writeColumn(TblColRef col, String value, int roundingFlag, 
byte defaultValue, byte[] output, int outputOffset) {
+DimensionEncoding dimEnc = dimEncMap.get(col);
+if (dimEnc instanceof DictionaryDimEnc)
+dimEnc = ((DictionaryDimEnc) dimEnc).copy(roundingFlag, 
defaultValue);
+
+dimEnc.encode(value, output, outputOffset);
+}
+
+public

[9/9] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-23 Thread liyang 
KYLIN-2195 All code changes, ready for test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/826f23f1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/826f23f1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/826f23f1

Branch: refs/heads/master
Commit: 826f23f14f60ba068b8594ba312c14e66c4807f2
Parents: 1ec1ffc
Author: Li Yang 
Authored: Wed Nov 23 11:49:20 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 23 17:08:29 2016 +0800

--
 build/bin/check-env.sh  |2 +-
 build/bin/find-hive-dependency.sh   |4 +-
 build/bin/sample.sh |8 +-
 build/conf/kylin.properties |   94 +-
 .../common/BackwardCompatibilityConfig.java |   21 +-
 .../apache/kylin/common/KylinConfigBase.java|  272 ++---
 .../common/persistence/JsonSerializer.java  |   94 +-
 .../kylin/common/persistence/Serializer.java|   70 +-
 .../common/persistence/StorageException.java|   76 +-
 .../kylin/common/restclient/RestClient.java |  222 ++--
 .../org/apache/kylin/common/util/Array.java |  164 +--
 .../org/apache/kylin/common/util/ByteArray.java |  492 -
 .../kylin/common/util/BytesSerializer.java  |   70 +-
 .../apache/kylin/common/util/BytesSplitter.java |  250 ++---
 .../kylin/common/util/CliCommandExecutor.java   |  298 ++---
 .../apache/kylin/common/util/MailService.java   |2 +-
 .../apache/kylin/common/util/ShardingHash.java  |  134 +--
 .../kylin-backward-compatibility.properties |  358 +++---
 .../apache/kylin/common/KylinConfigTest.java|2 +-
 .../kylin/common/util/MailServiceTest.java  |2 +-
 .../cube/common/FuzzyValueCombination.java  |  260 ++---
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java |  286 ++---
 .../org/apache/kylin/cube/kv/RowConstants.java  |   94 +-
 .../apache/kylin/cube/kv/RowKeyColumnIO.java|  126 +--
 .../apache/kylin/cube/kv/RowKeyColumnOrder.java |  216 ++--
 .../org/apache/kylin/cube/model/CubeDesc.java   |2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |  322 +++---
 .../kylin/cube/model/v1_4_0/RowKeyColDesc.java  |  184 ++--
 .../cube/model/validation/IValidatorRule.java   |   54 +-
 .../cube/model/validation/ResultLevel.java  |   76 +-
 .../cube/model/validation/ValidateContext.java  |  200 ++--
 .../validation/rule/AggregationGroupRule.java   |  400 +++
 .../model/validation/rule/FunctionRule.java |  404 +++
 .../model/validation/rule/RowKeyAttrRule.java   |  110 +-
 .../validation/rule/StreamingCubeRule.java  |  158 +--
 .../kylin/cube/AggregationGroupRuleTest.java|  244 ++---
 .../org/apache/kylin/cube/CubeDescTest.java |6 +-
 .../apache/kylin/cube/CubeManagerCacheTest.java |  156 +--
 .../apache/kylin/cube/RowKeyAttrRuleTest.java   |  120 +--
 .../metadata/measure/MeasureCodecTest.java  |  182 ++--
 .../kylin/dict/DictionaryInfoSerializer.java|  160 +--
 .../kylin/dict/TrieDictionaryBuilder.java   | 1020 +-
 .../kylin/dict/lookup/LookupStringTable.java|  220 ++--
 .../apache/kylin/dict/lookup/LookupTable.java   |  356 +++---
 .../kylin/dict/lookup/SnapshotManager.java  |  370 +++
 .../apache/kylin/dict/lookup/SnapshotTable.java |  570 +-
 .../dict/lookup/SnapshotTableSerializer.java|  158 +--
 .../kylin/dict/AppendTrieDictionaryTest.java|2 +-
 .../impl/threadpool/DistributedScheduler.java   |6 +-
 .../kylin/metadata/MetadataConstants.java   |  104 +-
 .../kylin/metadata/cachesync/Broadcaster.java   |2 +-
 .../kylin/metadata/model/JoinTableDesc.java |  124 +--
 .../org/apache/kylin/metadata/tuple/ITuple.java |   86 +-
 .../apache/kylin/storage/StorageContext.java|2 +-
 .../translate/DerivedFilterTranslator.java  |  400 +++
 .../translate/FuzzyValueCombination.java|  264 ++---
 .../apache/kylin/engine/mr/DFSFileTable.java|  198 ++--
 .../kylin/engine/mr/DFSFileTableReader.java |  504 -
 .../engine/mr/steps/FactDistinctColumnsJob.java |  318 +++---
 .../apache/kylin/engine/mr/TableReaderTest.java |   92 +-
 .../test_case_data/localmeta/kylin.properties   |   52 +-
 .../test_case_data/localmeta/kylin_job_conf.xml |  134 +--
 examples/test_case_data/sandbox/hive-site.xml   |6 +-
 .../test_case_data/sandbox/kylin.properties |   64 +-
 .../test_case_data/sandbox/kylin_job_conf.xml   |  156 +--
 .../sandbox/kylin_job_conf_inmem.xml|  182 ++--
 .../resources/org-apache-kylin-jdbc.properties  |   54 +-
 .../kylin/job/BaseTestDistributedScheduler.java |4 +-
 .../kylin/provision/BuildCubeWithEngine.java|2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|4 +-
 kylin-it/src/test/resources/logging.properties  |   44 +-
 .../aggregate/DimCountDistinctAggFuncTest.java  |4 +-

[5/9] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-23 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 916c369..598865b 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -1,512 +1,512 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.IdentityHashMap;
-import java.util.LinkedList;
-
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- * Builds a dictionary using Trie structure. All values are taken in byte[] 
form
- * and organized in a Trie with ordering. Then numeric IDs are assigned in
- * sequence.
- * 
- * @author yangli9
- */
-public class TrieDictionaryBuilder {
-
-private static final int _2GB = 20;
-
-public static class Node {
-public byte[] part;
-public boolean isEndOfValue;
-public ArrayList children;
-
-public int nValuesBeneath; // only present after stats()
-
-Node(byte[] value, boolean isEndOfValue) {
-reset(value, isEndOfValue);
-}
-
-Node(byte[] value, boolean isEndOfValue, ArrayList children) {
-reset(value, isEndOfValue, children);
-}
-
-void reset(byte[] value, boolean isEndOfValue) {
-reset(value, isEndOfValue, new ArrayList());
-}
-
-void reset(byte[] value, boolean isEndOfValue, ArrayList 
children) {
-this.part = value;
-this.isEndOfValue = isEndOfValue;
-this.children = children;
-}
-}
-
-public static interface Visitor {
-void visit(Node n, int level);
-}
-
-// 

-
-private Node root;
-private BytesConverter bytesConverter;
-
-public TrieDictionaryBuilder(BytesConverter bytesConverter) {
-this.root = new Node(new byte[0], false);
-this.bytesConverter = bytesConverter;
-}
-
-public void addValue(T value) {
-addValue(bytesConverter.convertToBytes(value));
-}
-
-// add a converted value (given in byte[] format), use with care, for 
internal only
-void addValue(byte[] value) {
-addValueR(root, value, 0);
-}
-
-private void addValueR(Node node, byte[] value, int start) {
-// match the value part of current node
-int i = 0, j = start;
-int n = node.part.length, nn = value.length;
-int comp = 0;
-for (; i < n && j < nn; i++, j++) {
-comp = BytesUtil.compareByteUnsigned(node.part[i], value[j]);
-if (comp != 0)
-break;
-}
-
-if (j == nn) {
-// if value fully matched within the current node
-if (i == n) {
-// if equals to current node, just mark end of value
-node.isEndOfValue = true;
-} else {
-// otherwise, split the current node into two
-Node c = new Node(BytesUtil.subarray(node.part, i, n), 
node.isEndOfValue, node.children);
-node.reset(BytesUtil.subarray(node.part, 0, i), true);
-node.children.add(c);
-}
-return;
-}
-
-// if partially matched the current, split the current node, add the 
new value, make a 3-way
-if (i < n) {
-Node c1 = new Node(BytesUtil.subarray(node.part, i, n), 
node.isEndOfValue, node.children);
-Node c2 = new Node(BytesUtil.subarray(value, j, nn), true);
-node.reset(BytesUtil.subarray(node.part, 0, i), false);
-if (comp < 0) {
-

[8/9] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-23 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index db02f34..db9fc80 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -1,246 +1,246 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-
-/**
- * @author yangli9
- */
-public class ByteArray implements Comparable, Serializable {
-
-private static final long serialVersionUID = 1L;
-
-public static final ByteArray EMPTY = new ImmutableByteArray();
-
-public static ByteArray allocate(int length) {
-return new ByteArray(new byte[length]);
-}
-
-public static ByteArray copyOf(byte[] array, int offset, int length) {
-byte[] space = new byte[length];
-System.arraycopy(array, offset, space, 0, length);
-return new ByteArray(space, 0, length);
-}
-
-// 

-
-private byte[] data;
-private int offset;
-private int length;
-
-public ByteArray() {
-this(null, 0, 0);
-}
-
-public ByteArray(int capacity) {
-this(new byte[capacity], 0, capacity);
-}
-
-public ByteArray(byte[] data) {
-this(data, 0, data == null ? 0 : data.length);
-}
-
-public ByteArray(byte[] data, int offset, int length) {
-this.data = data;
-this.offset = offset;
-this.length = length;
-}
-
-public byte[] array() {
-return data;
-}
-
-public int offset() {
-return offset;
-}
-
-public int length() {
-return length;
-}
-
-public void set(byte[] array) {
-set(array, 0, array.length);
-}
-
-public void set(byte[] array, int offset, int length) {
-this.data = array;
-this.offset = offset;
-this.length = length;
-}
-
-public void set(ByteArray o) {
-set(o.data, o.offset, o.length);
-}
-
-public void set(int offset, int length) {
-this.offset = offset;
-this.length = length;
-}
-
-public void setLength(int length) {
-this.length = length;
-}
-
-public ByteArray copy() {
-ByteArray copy;
-if (data != null) {
-copy = new ByteArray(length);
-} else {
-copy = new ByteArray(null);
-}
-copy.copyFrom(this);
-return copy;
-}
-
-//notice this will have a length header
-public void exportData(ByteBuffer out) {
-BytesUtil.writeByteArray(this.data, this.offset, this.length, out);
-}
-
-public static ByteArray importData(ByteBuffer in) {
-byte[] bytes = BytesUtil.readByteArray(in);
-return new ByteArray(bytes);
-}
-
-public void copyFrom(ByteArray other) {
-if (other.data != null) {
-System.arraycopy(other.array(), other.offset, data, offset, 
other.length);
-}
-this.length = other.length;
-}
-
-public ByteBuffer asBuffer() {
-if (data == null)
-return null;
-else if (offset == 0 && length == data.length)
-return ByteBuffer.wrap(data);
-else
-return ByteBuffer.wrap(data, offset, length).slice();
-}
-
-public byte[] toBytes() {
-return Bytes.copy(this.array(), this.offset(), this.length());
-}
-
-@Override
-public int hashCode() {
-if (data == null) {
-return 0;
-} else {
-if (length <= Bytes.SIZEOF_LONG && length > 0) {
-// to avoid hash collision of byte arrays those are converted 
from nearby integers/longs, which is the case for kylin dictionary
-long value = BytesUtil.readLong(data, offset, length);
-return (int) (value ^ (value

[4/9] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-23 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index db1a170..f2edcb4 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -1,285 +1,285 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.File;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import com.google.common.base.Strings;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.StringBytesConverter;
-import org.apache.kylin.dict.TrieDictionary;
-import org.apache.kylin.dict.TrieDictionaryBuilder;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.source.ReadableTable;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author yangli9
- */
-@SuppressWarnings("serial")
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
-public class SnapshotTable extends RootPersistentEntity implements 
ReadableTable {
-
-@JsonProperty("tableName")
-private String tableName;
-@JsonProperty("signature")
-private TableSignature signature;
-@JsonProperty("useDictionary")
-private boolean useDictionary;
-
-private ArrayList rowIndices;
-private Dictionary dict;
-
-// default constructor for JSON serialization
-public SnapshotTable() {
-}
-
-SnapshotTable(ReadableTable table, String tableName) throws IOException {
-this.tableName = tableName;
-this.signature = table.getSignature();
-this.useDictionary = true;
-}
-
-public void takeSnapshot(ReadableTable table, TableDesc tableDesc) throws 
IOException {
-this.signature = table.getSignature();
-
-int maxIndex = tableDesc.getMaxColumnIndex();
-
-TrieDictionaryBuilder b = new 
TrieDictionaryBuilder(new StringBytesConverter());
-
-TableReader reader = table.getReader();
-try {
-while (reader.next()) {
-String[] row = reader.getRow();
-if (row.length <= maxIndex) {
-throw new IllegalStateException("Bad hive table row, " + 
tableDesc + " expect " + (maxIndex + 1) + " columns, but got " + 
Arrays.toString(row));
-}
-for (ColumnDesc column : tableDesc.getColumns()) {
-String cell = row[column.getZeroBasedIndex()];
-if (cell != null)
-b.addValue(cell);
-}
-}
-} finally {
-IOUtils.closeQuietly(reader);
-}
-
-this.dict = b.build(0);
-
-ArrayList allRowIndices = new ArrayList();
-reader = table.getReader();
-try {
-while (reader.next()) {
-String[] row = reader.getRow();
-int[] rowIndex = new int[tableDesc.getColumnCount()];
-for (ColumnDesc column : tableDesc.getColumns()) {
-rowIndex[column.getZeroBasedIndex()] = 
dict.getIdFromValue(row[column.getZeroBasedIndex()]);
-}
-allRowIndices.add(rowIndex);
-}
-} finally {
-

[2/9] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-23 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/source-hive/src/test/java/org/apache/kylin/source/hive/HiveCmdBuilderTest.java
--
diff --git 
a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveCmdBuilderTest.java
 
b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveCmdBuilderTest.java
index da0c082..0eb02b4 100644
--- 
a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveCmdBuilderTest.java
+++ 
b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveCmdBuilderTest.java
@@ -40,13 +40,13 @@ public class HiveCmdBuilderTest {
 
 @After
 public void after() throws Exception {
-System.clearProperty("kylin.hive.client");
-System.clearProperty("kylin.hive.beeline.params");
+System.clearProperty("kylin.source.hive.client");
+System.clearProperty("kylin.source.hive.beeline-params");
 }
 
 @Test
 public void testHiveCLI() {
-System.setProperty("kylin.hive.client", "cli");
+System.setProperty("kylin.source.hive.client", "cli");
 
 HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
 hiveCmdBuilder.addStatement("USE default;");
@@ -59,8 +59,8 @@ public class HiveCmdBuilderTest {
 @Test
 public void testBeeline() throws IOException {
 String lineSeparator = java.security.AccessController.doPrivileged(new 
sun.security.action.GetPropertyAction("line.separator"));
-System.setProperty("kylin.hive.client", "beeline");
-System.setProperty("kylin.hive.beeline.params", "-u jdbc_url");
+System.setProperty("kylin.source.hive.client", "beeline");
+System.setProperty("kylin.source.hive.beeline-params", "-u jdbc_url");
 
 HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
 hiveCmdBuilder.addStatement("USE default;");

http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
index 3033bfd..f4d54c5 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
@@ -1,164 +1,164 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.source.kafka.hadoop;
-
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.source.kafka.util.KafkaClient;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.source.kafka.KafkaConfigManager;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * Run a Hadoop Job to process the stream data in kafka;
- * Modified from the kafka-hadoop-loader in 
https://github.com/amient/kafka-hadoop-loader
- */
-public class KafkaFlatTableJob extends AbstractHadoopJob {
-protected static final Logger logger = 
LoggerFactory.getLogger(KafkaFlatTableJob.class);
-
-public static final String CONFIG_KAFKA_PARITION_MIN = 
"kafka.partition.min";
-public static final String CONFIG_KAFKA_PARITION_MAX = 
"kafka.partition.max";
-public static final String CONFIG_KAFKA_PARITION_START = 
"kafka.partition.start.";
-public static

[3/9] kylin git commit: KYLIN-2195 All code changes, ready for test

2016-11-23 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
index 173c908..67fedbd 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
@@ -1,252 +1,252 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr;
-
-import java.io.BufferedReader;
-import java.io.Closeable;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.StringSplitter;
-import org.apache.kylin.source.ReadableTable.TableReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Tables are typically CSV or SEQ file.
- * 
- * @author yangli9
- */
-public class DFSFileTableReader implements TableReader {
-
-private static final Logger logger = 
LoggerFactory.getLogger(DFSFileTableReader.class);
-private static final char CSV_QUOTE = '"';
-private static final String[] DETECT_DELIMS = new String[] { "\177", "|", 
"\t", "," };
-
-private String filePath;
-private String delim;
-private List readerList;
-
-private String curLine;
-private String[] curColumns;
-private int expectedColumnNumber = -1; // helps delimiter detection
-
-public DFSFileTableReader(String filePath, int expectedColumnNumber) 
throws IOException {
-this(filePath, DFSFileTable.DELIM_AUTO, expectedColumnNumber);
-}
-
-public DFSFileTableReader(String filePath, String delim, int 
expectedColumnNumber) throws IOException {
-filePath = HadoopUtil.fixWindowsPath(filePath);
-this.filePath = filePath;
-this.delim = delim;
-this.expectedColumnNumber = expectedColumnNumber;
-this.readerList = new ArrayList();
-
-FileSystem fs = HadoopUtil.getFileSystem(filePath);
-
-ArrayList allFiles = new ArrayList<>();
-FileStatus status = fs.getFileStatus(new Path(filePath));
-if (status.isFile()) {
-allFiles.add(status);
-} else {
-FileStatus[] listStatus = fs.listStatus(new Path(filePath));
-allFiles.addAll(Arrays.asList(listStatus));
-}
-
-try {
-for (FileStatus f : allFiles) {
-RowReader rowReader = new 
SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, f.getPath().toString());
-this.readerList.add(rowReader);
-}
-} catch (IOException e) {
-if (isExceptionSayingNotSeqFile(e) == false)
-throw e;
-
-this.readerList = new ArrayList();
-for (FileStatus f : allFiles) {
-RowReader rowReader = new CsvRowReader(fs, 
f.getPath().toString());
-this.readerList.add(rowReader);
-}
-}
-}
-
-private boolean isExceptionSayingNotSeqFile(IOException e) {
-if (e.getMessage() != null && e.getMessage().contains("not a 
SequenceFile"))
-return true;
-
-if (e instanceof EOFException) // in case the file is very very small
-return true;
-
-return false;
-}
-
-@Override
-public boolean next() throws IOException {
-int curReaderIndex = -1;
-

[1/9] kylin git commit: KYLIN-2195 Rename spring/security related properties

2016-11-23 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 1ec1ffccd -> 2cca348c6


KYLIN-2195 Rename spring/security related properties


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2cca348c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2cca348c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2cca348c

Branch: refs/heads/master
Commit: 2cca348c608aa8674572ee6bee53fbf48e40e4df
Parents: 826f23f
Author: Li Yang 
Authored: Wed Nov 23 11:55:48 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 23 17:08:29 2016 +0800

--
 build/conf/kylin.properties | 179 ---
 .../common/BackwardCompatibilityConfig.java |  13 +-
 .../org/apache/kylin/common/KylinConfig.java|   6 +-
 .../apache/kylin/common/KylinConfigBase.java|  14 +-
 .../org/apache/kylin/common/KylinConfigExt.java |   4 +-
 .../kylin-backward-compatibility.properties |  37 +++-
 .../apache/kylin/common/KylinConfigTest.java|  18 +-
 .../test_case_data/localmeta/kylin.properties   |  34 ++--
 examples/test_case_data/sandbox/hive-site.xml   |   6 +-
 .../test_case_data/sandbox/kylin.properties |  13 +-
 server/src/main/resources/kylinSecurity.xml |  46 ++---
 webapp/app/js/controllers/cubes.js  |   2 +-
 webapp/app/js/services/kylinProperties.js   |   8 +-
 13 files changed, 193 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2cca348c/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 7cdc1c5..1409394 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -15,21 +15,32 @@
 # limitations under the License.
 #
 
-### SERVICE ###
+### METADATA | ENV ###
+
+# The metadata store in hbase
+kylin.metadata.url=kylin_metadata@hbase
+
+# Working folder in HDFS, make sure user has the right access to the hdfs 
directory
+kylin.env.hdfs-working-dir=/kylin
+
+# DEV|QA|PROD. DEV will turn on some dev features, QA and PROD has no 
difference in terms of functions.
+kylin.env=QA
+
+
+### SERVER | WEB ###
 
 # Kylin server mode, valid value [all, query, job]
 kyin.server.mode=all
 
-# Optional information for the owner of kylin platform, it can be your team's 
email
-# Currently it will be attached to each kylin's htable attribute
-kylin.storage.hbase.owner-tag=who...@kylin.apache.org
-
 # List of web servers in use, this enables one web server instance to sync up 
with other servers.
 kylin.server.cluster-servers=localhost:7070
 
 # Display timezone on UI,format like[GMT+N or GMT-N]
 kylin.web.timezone=GMT+8
 
+kylin.web.cross-domain-enabled=true
+
+
 ### SOURCE ###
 
 # Hive client, valid value [cli, beeline]
@@ -40,16 +51,17 @@ kylin.source.hive.client=cli
 
 kylin.source.hive.keep-flat-table=false
 
-### STORAGE ###
+# Hive database name for putting the intermediate flat tables
+kylin.source.hive.database-for-flat-table=default
 
-# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
+# Whether redistribute the intermediate flat table before building
+kylin.source.hive.redistribute-flat-table=true
 
-# The storage for final cube file in hbase
-kylin.storage.url=hbase
 
-# Working folder in HDFS, make sure user has the right access to the hdfs 
directory
-kylin.env.hdfs-working-dir=/kylin
+### STORAGE ###
+
+# The storage for cube is hbase
+kylin.storage.url=hbase
 
 # Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]
 kylin.storage.hbase.compression-codec=none
@@ -68,72 +80,67 @@ kylin.storage.hbase.hfile-size-gb=2
 kylin.storage.hbase.min-region-count=1
 kylin.storage.hbase.max-region-count=500
 
-### JOB ###
-
-# max job retry on error, default 0: no retry
-kylin.job.retry=0
+# Optional information for the owner of kylin platform, it can be your team's 
email
+# Currently it will be attached to each kylin's htable attribute
+kylin.storage.hbase.owner-tag=who...@kylin.apache.org
 
-# If true, job engine will not assume that hadoop CLI reside on the same 
server as it self
-# you will have to specify kylin.job.remote-cli-hostname, 
kylin.job.remote-cli-username and kylin.job.remote-cli-password
-# It should not be set to "true" unless you're NOT running Kylin.sh on a 
hadoop client machine 
-# (Thus kylin instance has to ssh to another real hadoop client machine to 
execute hbase,hive,hadoop commands)
-kylin.job.use-remote-cli=false
+kylin.storage.hbase.coprocessor-mem-gb=3
 
-# Only necessary when kylin.job.use-remote-cli=true
-kylin.job.remote-cli-hostname=
-kylin.job.remote-cli-port=22
+# The default coprocessor timeout is (hbase.rpc.timeout * 0.9) / 1000 seconds,
+# You can set it to a smaller value. 0 means

svn commit: r17136 - in /dev/kylin: apache-kylin-0.7.2-incubating-rc2/ apache-kylin-1.0-incubating-rc2/ apache-kylin-1.1-incubating-rc1/ apache-kylin-1.2-rc1/

2016-11-23 Thread shaofengshi 
Author: shaofengshi
Date: Wed Nov 23 09:03:54 2016
New Revision: 17136

Log:
remoev old artifacts

Removed:
dev/kylin/apache-kylin-0.7.2-incubating-rc2/
dev/kylin/apache-kylin-1.0-incubating-rc2/
dev/kylin/apache-kylin-1.1-incubating-rc1/
dev/kylin/apache-kylin-1.2-rc1/

kylin git commit: KYLIN-2224 fix this issue

2016-11-23 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 53b5a2752 -> 1ec1ffccd


KYLIN-2224 fix this issue

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1ec1ffcc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1ec1ffcc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1ec1ffcc

Branch: refs/heads/master
Commit: 1ec1ffccd787036cbc89d0a549adcbb0469cb1ec
Parents: 53b5a27
Author: Cheng Wang 
Authored: Wed Nov 23 11:19:04 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 23 11:46:35 2016 +0800

--
 .../main/java/org/apache/kylin/cube/RawQueryLastHacker.java  | 5 +++--
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 2 +-
 .../apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java | 8 
 3 files changed, 8 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1ec1ffcc/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index 50c644e..b0a4823 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -24,6 +24,7 @@ import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.TupleInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,7 +32,7 @@ public class RawQueryLastHacker {
 
 private static final Logger logger = 
LoggerFactory.getLogger(RawQueryLastHacker.class);
 
-public static void hackNoAggregations(SQLDigest sqlDigest, CubeDesc 
cubeDesc) {
+public static void hackNoAggregations(SQLDigest sqlDigest, CubeDesc 
cubeDesc, TupleInfo tupleInfo) {
 if (!sqlDigest.isRawQuery) {
 return;
 }
@@ -45,7 +46,7 @@ public class RawQueryLastHacker {
 boolean isSelectAll = sqlDigest.allColumns.isEmpty() || 
sqlDigest.allColumns.equals(sqlDigest.filterColumns);
 for (TblColRef col : cubeDesc.listAllColumns()) {
 if (cubeDesc.listDimensionColumnsIncludingDerived().contains(col) 
|| isSelectAll) {
-if (col.getTable().equals(sqlDigest.factTable))
+if (tupleInfo.hasColumn(col))
 sqlDigest.allColumns.add(col);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1ec1ffcc/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 24dd2fb..9c74cca 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -77,7 +77,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 sqlDigest.includeSubqueryJoinParticipants();
 
 //cope with queries with no aggregations
-RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
+RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc, 
returnTupleInfo);
 
 // Customized measure taking effect: e.g. allow custom measures to 
help raw queries
 notifyBeforeStorageQuery(sqlDigest);

http://git-wip-us.apache.org/repos/asf/kylin/blob/1ec1ffcc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 3c73dc6..9af0faf 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -46,10 +46,10 @@ import org.apache.kylin.cube.RawQueryLastHacker;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
 import org.apache.kylin.cube.model.HBaseColumnDesc;