[4/8] kylin git commit: KYLIN-2217 fix bug

2016-11-24 Thread liyang 
KYLIN-2217 fix bug


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d9c8d9ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d9c8d9ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d9c8d9ec

Branch: refs/heads/KYLIN-2217-2
Commit: d9c8d9ec26b910f2a27f7a98dd44ef49e61fe416
Parents: 7f7417b
Author: Yang Li 
Authored: Fri Nov 25 07:16:45 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 15:05:36 2016 +0800

--
 .../main/java/org/apache/kylin/engine/mr/DFSFileTable.java| 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d9c8d9ec/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
index 8c1f6bd..6f337d0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
@@ -61,12 +61,13 @@ public class DFSFileTable implements ReadableTable {
 
 @Override
 public TableSignature getSignature() throws IOException {
+Pair sizeAndLastModified;
 try {
-Pair sizeAndLastModified = 
getSizeAndLastModified(path);
-return new TableSignature(path, sizeAndLastModified.getFirst(), 
sizeAndLastModified.getSecond());
+sizeAndLastModified = getSizeAndLastModified(path);
 } catch (FileNotFoundException ex) {
-return null;
+sizeAndLastModified = Pair.newPair(-1L, 0L);
 }
+return new TableSignature(path, sizeAndLastModified.getFirst(), 
sizeAndLastModified.getSecond());
 }
 
 @Override

[3/8] kylin git commit: minor, keep same column in Rowkey and AggrGroup

2016-11-24 Thread liyang 
minor, keep same column in Rowkey and AggrGroup


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d367bf56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d367bf56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d367bf56

Branch: refs/heads/KYLIN-2217-2
Commit: d367bf56df8cf6581cc3dbdae57b5ed167be47f7
Parents: 89875fa
Author: Li Yang 
Authored: Fri Nov 25 15:05:19 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 15:05:19 2016 +0800

--
 .../java/org/apache/kylin/cube/model/AggregationGroup.java| 2 +-
 .../main/java/org/apache/kylin/cube/model/RowKeyColDesc.java  | 2 +-
 .../main/java/org/apache/kylin/metadata/model/TblColRef.java  | 7 +++
 3 files changed, 9 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d367bf56/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 12c0574..9bd082f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -107,7 +107,7 @@ public class AggregationGroup {
 
 for (int i = 0; i < names.length; i++) {
 TblColRef col = cubeDesc.getModel().findColumn(names[i]);
-names[i] = col.getTableAlias() + "." + col.getName();
+names[i] = col.getIdentity();
 }
 
 // check no dup

http://git-wip-us.apache.org/repos/asf/kylin/blob/d367bf56/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 326052e..3b49323 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -62,9 +62,9 @@ public class RowKeyColDesc {
 private TblColRef colRef;
 
 public void init(int index, CubeDesc cubeDesc) {
-column = column.toUpperCase();
 bitIndex = index;
 colRef = cubeDesc.getModel().findColumn(column);
+column = colRef.getIdentity();
 Preconditions.checkArgument(colRef != null, "Cannot find rowkey column 
%s in cube %s", column, cubeDesc);
 
 Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));

http://git-wip-us.apache.org/repos/asf/kylin/blob/d367bf56/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 9b71524..2cfbafc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -99,6 +99,7 @@ public class TblColRef implements Serializable {
 
 private TableRef table;
 private ColumnDesc column;
+private String identity;
 
 TblColRef(ColumnDesc column) {
 this.column = column;
@@ -188,6 +189,12 @@ public class TblColRef implements Serializable {
 return true;
 }
 
+public String getIdentity() {
+if (identity == null)
+identity = getTableAlias() + "." + getName();
+return identity;
+}
+
 @Override
 public String toString() {
 String alias = table == null ? "UNKNOWN_MODEL" : table.getAlias();

[7/8] kylin git commit: KYLIN-2220 Enforce same name between Cube & CubeDesc

2016-11-24 Thread liyang 
KYLIN-2220 Enforce same name between Cube & CubeDesc

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b078dd9f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b078dd9f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b078dd9f

Branch: refs/heads/KYLIN-2217-2
Commit: b078dd9f1b8ce9b7d0dad85cc6c9aa55dd5d9b15
Parents: d367bf5
Author: kangkaisen 
Authored: Tue Nov 22 21:26:32 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 15:05:36 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java  | 19 +--
 .../kylin/rest/controller/CubeController.java|  2 +-
 .../apache/kylin/rest/service/CubeService.java   |  8 
 3 files changed, 2 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b078dd9f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 9893040..307bb46 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -617,24 +617,6 @@ public class CubeManager implements IRealizationProvider {
 }
 }
 
-private long calculateStartOffsetForAppendSegment(CubeInstance cube) {
-List existing = cube.getSegments();
-if (existing.isEmpty()) {
-return 0;
-} else {
-return existing.get(existing.size() - 1).getSourceOffsetEnd();
-}
-}
-
-private long calculateStartDateForAppendSegment(CubeInstance cube) {
-List existing = cube.getSegments();
-if (existing.isEmpty()) {
-return cube.getDescriptor().getPartitionDateStart();
-} else {
-return existing.get(existing.size() - 1).getDateRangeEnd();
-}
-}
-
 private void checkBuildingSegment(CubeInstance cube) {
 int maxBuldingSeg = cube.getConfig().getMaxBuildingSegments();
 if (cube.getBuildingSegments().size() >= maxBuldingSeg) {
@@ -923,6 +905,7 @@ public class CubeManager implements IRealizationProvider {
 
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", cube.getDescName(), cubeName);
+checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' 
must be same as descriptor name '%s', but it is not", cubeName, 
cubeDesc.getName());
 
 if (!cubeDesc.getError().isEmpty()) {
 cube.setStatus(RealizationStatusEnum.DESCBROKEN);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b078dd9f/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index e1aa17a..3846d28 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -416,7 +416,7 @@ public class CubeController extends BasicController {
 cubeRequest.setMessage("CubeDesc is null.");
 return cubeRequest;
 }
-String name = CubeService.getCubeNameFromDesc(desc.getName());
+String name = desc.getName();
 if (StringUtils.isEmpty(name)) {
 logger.info("Cube name should not be empty.");
 throw new BadRequestException("Cube name should not be empty.");

http://git-wip-us.apache.org/repos/asf/kylin/blob/b078dd9f/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 5c59e1a..85c9284 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -278,14 +278,6 @@ public class CubeService extends BasicService {
 accessService.clean(cube, true);
 }
 
-public static String getCubeNameFromDesc(String descName) {
-if (descName.toLowerCase().endsWith(DESC_SUFFIX)) {
-return descName.substring(0, 
descName.toLowerCase().indexOf(DESC_SUFFIX));
-} else {
-

[1/8] kylin git commit: minor, trim build log [Forced Update!]

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 84a0726a5 -> 99197dafc (forced update)


minor, trim build log

Revert "KYLIN-2210 call CubeStatsReader.print at SaveStatisticsStep"

This reverts commit 3ab966b650debb83eb219c9ed8d357d141466776.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8530ebd6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8530ebd6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8530ebd6

Branch: refs/heads/KYLIN-2217-2
Commit: 8530ebd692b3362f7fb9eb07160c8c069d467fc4
Parents: 6afcb26
Author: Hongbin Ma 
Authored: Fri Nov 25 09:52:47 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 25 09:58:10 2016 +0800

--
 .../kylin/engine/mr/common/CubeStatsReader.java|  2 +-
 .../kylin/engine/mr/steps/SaveStatisticsStep.java  | 17 +++--
 2 files changed, 4 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8530ebd6/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c6839d6..c917cfb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -227,7 +227,7 @@ public class CubeStatsReader {
 return ret;
 }
 
-public void print(PrintWriter out) {
+private void print(PrintWriter out) {
 Map cuboidRows = getCuboidRowEstimatesHLL();
 Map cuboidSizes = getCuboidSizeMap();
 List cuboids = new ArrayList(cuboidRows.keySet());

http://git-wip-us.apache.org/repos/asf/kylin/blob/8530ebd6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 79346a5..020c62c 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -19,8 +19,6 @@
 package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
 import java.util.Random;
 
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -31,8 +29,8 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
 import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.engine.mr.common.CubeStatsReader;
 import org.apache.kylin.job.exception.ExecuteException;
@@ -86,16 +84,6 @@ public class SaveStatisticsStep extends AbstractExecutable {
 
 private void decideCubingAlgorithm(CubeSegment seg, KylinConfig kylinConf) 
throws IOException {
 String algPref = kylinConf.getCubeAlgorithm();
-
-CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
-StringWriter sw = new StringWriter();
-PrintWriter pw = new PrintWriter(sw);
-cubeStats.print(pw);
-pw.flush();
-pw.close();
-logger.info("Cube Stats Estimation for segment {} :", seg.toString());
-logger.info(sw.toString());
-
 AlgorithmEnum alg;
 if (AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
 alg = AlgorithmEnum.INMEM;
@@ -115,13 +103,14 @@ public class SaveStatisticsStep extends 
AbstractExecutable {
 } else if ("random".equalsIgnoreCase(algPref)) { // for testing
 alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : 
AlgorithmEnum.LAYER;
 } else { // the default
+CubeStatsReader cubeStats = new CubeStatsReader(seg, 
kylinConf);
 int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
 int mapperNumLimit = 
kylinConf.getCubeAlgorithmAutoMapperLimit();
 double mapperOverlapRatio = 
cubeStats.getMapperOverlapRatioOfFirstBuild();
 double overlapThreshold = 
kylinConf.getCubeAlgorithmAutoThreshold();
 logger.info("mapperNumber for " + seg + " is " + mapperNumber 
+ " and threshold is " +

[2/8] kylin git commit: KYLIN-2195 re-format KylinConfigBase

2016-11-24 Thread liyang 
KYLIN-2195 re-format KylinConfigBase


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/89875fae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/89875fae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/89875fae

Branch: refs/heads/KYLIN-2217-2
Commit: 89875fae78556d4d5f5099accb70d6572faeb3d4
Parents: 8530ebd
Author: Li Yang 
Authored: Fri Nov 25 14:49:58 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 14:49:58 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java| 778 ++-
 1 file changed, 411 insertions(+), 367 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/89875fae/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6131013..8ea03be 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -142,16 +142,47 @@ abstract public class KylinConfigBase implements 
Serializable {
 this.properties = BCC.check(properties);
 }
 
+private Map convertKeyToInteger(Map map) {
+Map result = Maps.newLinkedHashMap();
+for (Entry entry : map.entrySet()) {
+result.put(Integer.valueOf(entry.getKey()), entry.getValue());
+}
+return result;
+}
+
+public String toString() {
+return getMetadataUrl();
+}
+
+// 

+// ENV
 // 

 
 public boolean isDevEnv() {
 return "DEV".equals(getOptional("kylin.env", "DEV"));
 }
+
+public String getDeployEnv() {
+return getOptional("kylin.env", "DEV");
+}
+
+public String getHdfsWorkingDirectory() {
+String root = getRequired("kylin.env.hdfs-working-dir");
+if (!root.endsWith("/")) {
+root += "/";
+}
+return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
+}
+
+// 

+// METADATA
+// 

 
 public String getMetadataUrl() {
 return getOptional("kylin.metadata.url");
 }
 
+// for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);
 }
@@ -169,178 +200,120 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 }
 
-public String getServerMode() {
-return this.getOptional("kylin.server.mode", "all");
-}
-
-public String getStorageUrl() {
-return getOptional("kylin.storage.url");
-}
-
-public void setStorageUrl(String storageUrl) {
-setProperty("kylin.storage.url", storageUrl);
+public String[] getRealizationProviders() {
+return getOptionalStringArray("kylin.metadata.realization-providers", 
//
+new String[] { "org.apache.kylin.cube.CubeManager", 
"org.apache.kylin.storage.hybrid.HybridManager" });
 }
 
-/**
- * was for route to hive, not used any more
- */
-@Deprecated
-public String getHiveUrl() {
-return getOptional("kylin.source.hive.connection-url", "");
+public String[] getCubeDimensionCustomEncodingFactories() {
+return 
getOptionalStringArray("kylin.metadata.custom-dimension-encodings", new 
String[0]);
 }
 
-/**
- * was for route to hive, not used any more
- */
-@Deprecated
-public String getHiveUser() {
-return getOptional("kylin.source.hive.connection-user", "");
+public Map getCubeCustomMeasureTypes() {
+return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
 }
+
+// 

+// DICTIONARY & SNAPSHOT
+// 

 
-/**
- * was for route to hive, not used any more
- */
-@Deprecated
-public String getHivePassword() {
-return getOptional("kylin.source.hive.connection-password", "");
+public int getTrieDictionaryForestMaxTrieSizeMB() {
+return 
Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb",

[6/8] kylin git commit: KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager

2016-11-24 Thread liyang 
KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7f7417bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7f7417bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7f7417bd

Branch: refs/heads/KYLIN-2217-2
Commit: 7f7417bd2ed0eaa1766597d9dd4131cbe1d29757
Parents: b078dd9
Author: Li Yang 
Authored: Thu Nov 24 17:24:57 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 15:05:36 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 29 ++---
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  | 34 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |  2 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |  5 ++
 .../apache/kylin/dict/DictionaryManager.java| 65 +---
 .../engine/mr/steps/MergeCuboidMapperTest.java  |  2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |  6 +-
 7 files changed, 93 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7f7417bd/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 307bb46..ea8ff81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -48,7 +48,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.DistinctColumnValuesProvider;
 import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
@@ -214,25 +213,39 @@ public class CubeManager implements IRealizationProvider {
 return result;
 }
 
-public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable) throws IOException {
 CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
 return null;
 
-DictionaryManager dictMgr = getDictionaryManager();
 String builderClass = cubeDesc.getDictionaryBuilderClass(col);
-DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), 
col, factTableValueProvider, builderClass);
+DictionaryInfo dictInfo = 
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, 
builderClass);
 
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable, Dictionary dict) throws IOException {
+CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
+return null;
+
+DictionaryInfo dictInfo = 
getDictionaryManager().saveDictionary(cubeDesc.getModel(), col, inpTable, dict);
+
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, 
DictionaryInfo dictInfo) throws IOException {
 if (dictInfo != null) {
 Dictionary dict = dictInfo.getDictionaryObject();
 cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
 cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), 
dict.getSize(), dict.getSizeOfId() });
 
-CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
-cubeBuilder.setToUpdateSegs(cubeSeg);
-updateCube(cubeBuilder);
+CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
+update.setToUpdateSegs(cubeSeg);
+updateCube(update);
 }
-return dictInfo;
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f7417bd/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 89e2e9b..a6aeb96 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++

[5/8] kylin git commit: bug fix

2016-11-24 Thread liyang 
bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/99197daf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/99197daf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/99197daf

Branch: refs/heads/KYLIN-2217-2
Commit: 99197dafc3e33bca922e0ce711f54956169c77d5
Parents: f700a42
Author: Li Yang 
Authored: Fri Nov 25 11:25:08 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 15:05:36 2016 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/99197daf/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 6a024b4..b4422d2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -949,7 +949,8 @@ public class CubeManager implements IRealizationProvider {
 }
 
 private boolean isITTestCube(String cubeName) {
-return cubeName.startsWith("test_kylin_cube") || 
cubeName.startsWith("test_streaming");
+return config.isDevEnv() //
+&& (cubeName.startsWith("test_kylin_cube") || 
cubeName.startsWith("test_streaming"));
 }
 
 private MetadataManager getMetadataManager() {

[kylin] Git Push Summary [forced push!] [Forced Update!]

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 672500a10 -> 84a0726a5 (forced update)

kylin git commit: minor, keep same column in Rowkey and AggrGroup

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 89875fae7 -> d367bf56d


minor, keep same column in Rowkey and AggrGroup


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d367bf56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d367bf56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d367bf56

Branch: refs/heads/master
Commit: d367bf56df8cf6581cc3dbdae57b5ed167be47f7
Parents: 89875fa
Author: Li Yang 
Authored: Fri Nov 25 15:05:19 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 15:05:19 2016 +0800

--
 .../java/org/apache/kylin/cube/model/AggregationGroup.java| 2 +-
 .../main/java/org/apache/kylin/cube/model/RowKeyColDesc.java  | 2 +-
 .../main/java/org/apache/kylin/metadata/model/TblColRef.java  | 7 +++
 3 files changed, 9 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d367bf56/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 12c0574..9bd082f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -107,7 +107,7 @@ public class AggregationGroup {
 
 for (int i = 0; i < names.length; i++) {
 TblColRef col = cubeDesc.getModel().findColumn(names[i]);
-names[i] = col.getTableAlias() + "." + col.getName();
+names[i] = col.getIdentity();
 }
 
 // check no dup

http://git-wip-us.apache.org/repos/asf/kylin/blob/d367bf56/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 326052e..3b49323 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -62,9 +62,9 @@ public class RowKeyColDesc {
 private TblColRef colRef;
 
 public void init(int index, CubeDesc cubeDesc) {
-column = column.toUpperCase();
 bitIndex = index;
 colRef = cubeDesc.getModel().findColumn(column);
+column = colRef.getIdentity();
 Preconditions.checkArgument(colRef != null, "Cannot find rowkey column 
%s in cube %s", column, cubeDesc);
 
 Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));

http://git-wip-us.apache.org/repos/asf/kylin/blob/d367bf56/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 9b71524..2cfbafc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -99,6 +99,7 @@ public class TblColRef implements Serializable {
 
 private TableRef table;
 private ColumnDesc column;
+private String identity;
 
 TblColRef(ColumnDesc column) {
 this.column = column;
@@ -188,6 +189,12 @@ public class TblColRef implements Serializable {
 return true;
 }
 
+public String getIdentity() {
+if (identity == null)
+identity = getTableAlias() + "." + getName();
+return identity;
+}
+
 @Override
 public String toString() {
 String alias = table == null ? "UNKNOWN_MODEL" : table.getAlias();

kylin git commit: KYLIN-2195 re-format KylinConfigBase

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 8530ebd69 -> 89875fae7


KYLIN-2195 re-format KylinConfigBase


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/89875fae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/89875fae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/89875fae

Branch: refs/heads/master
Commit: 89875fae78556d4d5f5099accb70d6572faeb3d4
Parents: 8530ebd
Author: Li Yang 
Authored: Fri Nov 25 14:49:58 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 14:49:58 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java| 778 ++-
 1 file changed, 411 insertions(+), 367 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/89875fae/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6131013..8ea03be 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -142,16 +142,47 @@ abstract public class KylinConfigBase implements 
Serializable {
 this.properties = BCC.check(properties);
 }
 
+private Map convertKeyToInteger(Map map) {
+Map result = Maps.newLinkedHashMap();
+for (Entry entry : map.entrySet()) {
+result.put(Integer.valueOf(entry.getKey()), entry.getValue());
+}
+return result;
+}
+
+public String toString() {
+return getMetadataUrl();
+}
+
+// 

+// ENV
 // 

 
 public boolean isDevEnv() {
 return "DEV".equals(getOptional("kylin.env", "DEV"));
 }
+
+public String getDeployEnv() {
+return getOptional("kylin.env", "DEV");
+}
+
+public String getHdfsWorkingDirectory() {
+String root = getRequired("kylin.env.hdfs-working-dir");
+if (!root.endsWith("/")) {
+root += "/";
+}
+return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
+}
+
+// 

+// METADATA
+// 

 
 public String getMetadataUrl() {
 return getOptional("kylin.metadata.url");
 }
 
+// for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);
 }
@@ -169,178 +200,120 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 }
 
-public String getServerMode() {
-return this.getOptional("kylin.server.mode", "all");
-}
-
-public String getStorageUrl() {
-return getOptional("kylin.storage.url");
-}
-
-public void setStorageUrl(String storageUrl) {
-setProperty("kylin.storage.url", storageUrl);
+public String[] getRealizationProviders() {
+return getOptionalStringArray("kylin.metadata.realization-providers", 
//
+new String[] { "org.apache.kylin.cube.CubeManager", 
"org.apache.kylin.storage.hybrid.HybridManager" });
 }
 
-/**
- * was for route to hive, not used any more
- */
-@Deprecated
-public String getHiveUrl() {
-return getOptional("kylin.source.hive.connection-url", "");
+public String[] getCubeDimensionCustomEncodingFactories() {
+return 
getOptionalStringArray("kylin.metadata.custom-dimension-encodings", new 
String[0]);
 }
 
-/**
- * was for route to hive, not used any more
- */
-@Deprecated
-public String getHiveUser() {
-return getOptional("kylin.source.hive.connection-user", "");
+public Map getCubeCustomMeasureTypes() {
+return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
 }
+
+// 

+// DICTIONARY & SNAPSHOT
+// 

 
-/**
- * was for route to hive, not used any more
- */
-@Deprecated
-public String getHivePassword() {
-return getOptional("kylin.source.hive.connection-password", "");
+public int getTrieDictionaryForestMaxTrieSizeMB() {
+

kylin git commit: bug fix

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 84a0726a5 -> 672500a10


bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/672500a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/672500a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/672500a1

Branch: refs/heads/KYLIN-2217-2
Commit: 672500a1082ae8fec8257709bd4a1b58f3a47f01
Parents: 84a0726
Author: Li Yang 
Authored: Fri Nov 25 11:25:08 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 11:25:08 2016 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/672500a1/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 6a024b4..b4422d2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -949,7 +949,8 @@ public class CubeManager implements IRealizationProvider {
 }
 
 private boolean isITTestCube(String cubeName) {
-return cubeName.startsWith("test_kylin_cube") || 
cubeName.startsWith("test_streaming");
+return config.isDevEnv() //
+&& (cubeName.startsWith("test_kylin_cube") || 
cubeName.startsWith("test_streaming"));
 }
 
 private MetadataManager getMetadataManager() {

kylin git commit: fix bug

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 7b50d3163 -> 84a0726a5


fix bug


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/84a0726a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/84a0726a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/84a0726a

Branch: refs/heads/KYLIN-2217-2
Commit: 84a0726a562f1d2b7915795ec91eb83b44b15837
Parents: 7b50d31
Author: Li Yang 
Authored: Fri Nov 25 11:22:11 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 11:22:11 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeManager.java  | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/84a0726a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index ea8ff81..6a024b4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -918,7 +918,8 @@ public class CubeManager implements IRealizationProvider {
 
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", cube.getDescName(), cubeName);
-checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' 
must be same as descriptor name '%s', but it is not", cubeName, 
cubeDesc.getName());
+if (!isITTestCube(cubeName))
+checkState(cubeDesc.getName().equals(cubeName), "cube name 
'%s' must be same as descriptor name '%s', but it is not", cubeName, 
cubeDesc.getName());
 
 if (!cubeDesc.getError().isEmpty()) {
 cube.setStatus(RealizationStatusEnum.DESCBROKEN);
@@ -947,6 +948,10 @@ public class CubeManager implements IRealizationProvider {
 }
 }
 
+private boolean isITTestCube(String cubeName) {
+return cubeName.startsWith("test_kylin_cube") || 
cubeName.startsWith("test_streaming");
+}
+
 private MetadataManager getMetadataManager() {
 return MetadataManager.getInstance(config);
 }

kylin git commit: minor, trim build log

2016-11-24 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 6afcb2690 -> 8530ebd69


minor, trim build log

Revert "KYLIN-2210 call CubeStatsReader.print at SaveStatisticsStep"

This reverts commit 3ab966b650debb83eb219c9ed8d357d141466776.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8530ebd6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8530ebd6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8530ebd6

Branch: refs/heads/master
Commit: 8530ebd692b3362f7fb9eb07160c8c069d467fc4
Parents: 6afcb26
Author: Hongbin Ma 
Authored: Fri Nov 25 09:52:47 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 25 09:58:10 2016 +0800

--
 .../kylin/engine/mr/common/CubeStatsReader.java|  2 +-
 .../kylin/engine/mr/steps/SaveStatisticsStep.java  | 17 +++--
 2 files changed, 4 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8530ebd6/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c6839d6..c917cfb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -227,7 +227,7 @@ public class CubeStatsReader {
 return ret;
 }
 
-public void print(PrintWriter out) {
+private void print(PrintWriter out) {
 Map cuboidRows = getCuboidRowEstimatesHLL();
 Map cuboidSizes = getCuboidSizeMap();
 List cuboids = new ArrayList(cuboidRows.keySet());

http://git-wip-us.apache.org/repos/asf/kylin/blob/8530ebd6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 79346a5..020c62c 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -19,8 +19,6 @@
 package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
 import java.util.Random;
 
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -31,8 +29,8 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
 import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.engine.mr.common.CubeStatsReader;
 import org.apache.kylin.job.exception.ExecuteException;
@@ -86,16 +84,6 @@ public class SaveStatisticsStep extends AbstractExecutable {
 
 private void decideCubingAlgorithm(CubeSegment seg, KylinConfig kylinConf) 
throws IOException {
 String algPref = kylinConf.getCubeAlgorithm();
-
-CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
-StringWriter sw = new StringWriter();
-PrintWriter pw = new PrintWriter(sw);
-cubeStats.print(pw);
-pw.flush();
-pw.close();
-logger.info("Cube Stats Estimation for segment {} :", seg.toString());
-logger.info(sw.toString());
-
 AlgorithmEnum alg;
 if (AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
 alg = AlgorithmEnum.INMEM;
@@ -115,13 +103,14 @@ public class SaveStatisticsStep extends 
AbstractExecutable {
 } else if ("random".equalsIgnoreCase(algPref)) { // for testing
 alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : 
AlgorithmEnum.LAYER;
 } else { // the default
+CubeStatsReader cubeStats = new CubeStatsReader(seg, 
kylinConf);
 int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
 int mapperNumLimit = 
kylinConf.getCubeAlgorithmAutoMapperLimit();
 double mapperOverlapRatio = 
cubeStats.getMapperOverlapRatioOfFirstBuild();
 double overlapThreshold = 
kylinConf.getCubeAlgorithmAutoThreshold();
 logger.info("mapperNumber for " + seg + " is " + mapperNumber 
+ " and threshold is " + mapperNumLimit);

[3/3] kylin git commit: KYLIN-2217 fix bug

2016-11-24 Thread liyang 
KYLIN-2217 fix bug


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b50d316
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b50d316
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b50d316

Branch: refs/heads/KYLIN-2217-2
Commit: 7b50d3163456a748a98779b432161bde9da71b7d
Parents: 917a885
Author: Yang Li 
Authored: Fri Nov 25 07:16:45 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 09:48:33 2016 +0800

--
 .../main/java/org/apache/kylin/engine/mr/DFSFileTable.java| 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b50d316/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
index 8c1f6bd..6f337d0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
@@ -61,12 +61,13 @@ public class DFSFileTable implements ReadableTable {
 
 @Override
 public TableSignature getSignature() throws IOException {
+Pair sizeAndLastModified;
 try {
-Pair sizeAndLastModified = 
getSizeAndLastModified(path);
-return new TableSignature(path, sizeAndLastModified.getFirst(), 
sizeAndLastModified.getSecond());
+sizeAndLastModified = getSizeAndLastModified(path);
 } catch (FileNotFoundException ex) {
-return null;
+sizeAndLastModified = Pair.newPair(-1L, 0L);
 }
+return new TableSignature(path, sizeAndLastModified.getFirst(), 
sizeAndLastModified.getSecond());
 }
 
 @Override

[1/3] kylin git commit: KYLIN-2220 Enforce same name between Cube & CubeDesc [Forced Update!]

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 d14de80ee -> 7b50d3163 (forced update)


KYLIN-2220 Enforce same name between Cube & CubeDesc

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7fe7fc3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7fe7fc3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7fe7fc3e

Branch: refs/heads/KYLIN-2217-2
Commit: 7fe7fc3e36863c988ebdd9f34807291aac0aa556
Parents: 6afcb26
Author: kangkaisen 
Authored: Tue Nov 22 21:26:32 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 09:46:55 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java  | 19 +--
 .../kylin/rest/controller/CubeController.java|  2 +-
 .../apache/kylin/rest/service/CubeService.java   |  8 
 3 files changed, 2 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7fe7fc3e/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 9893040..307bb46 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -617,24 +617,6 @@ public class CubeManager implements IRealizationProvider {
 }
 }
 
-private long calculateStartOffsetForAppendSegment(CubeInstance cube) {
-List existing = cube.getSegments();
-if (existing.isEmpty()) {
-return 0;
-} else {
-return existing.get(existing.size() - 1).getSourceOffsetEnd();
-}
-}
-
-private long calculateStartDateForAppendSegment(CubeInstance cube) {
-List existing = cube.getSegments();
-if (existing.isEmpty()) {
-return cube.getDescriptor().getPartitionDateStart();
-} else {
-return existing.get(existing.size() - 1).getDateRangeEnd();
-}
-}
-
 private void checkBuildingSegment(CubeInstance cube) {
 int maxBuldingSeg = cube.getConfig().getMaxBuildingSegments();
 if (cube.getBuildingSegments().size() >= maxBuldingSeg) {
@@ -923,6 +905,7 @@ public class CubeManager implements IRealizationProvider {
 
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
 checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not 
found", cube.getDescName(), cubeName);
+checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' 
must be same as descriptor name '%s', but it is not", cubeName, 
cubeDesc.getName());
 
 if (!cubeDesc.getError().isEmpty()) {
 cube.setStatus(RealizationStatusEnum.DESCBROKEN);

http://git-wip-us.apache.org/repos/asf/kylin/blob/7fe7fc3e/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index e1aa17a..3846d28 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -416,7 +416,7 @@ public class CubeController extends BasicController {
 cubeRequest.setMessage("CubeDesc is null.");
 return cubeRequest;
 }
-String name = CubeService.getCubeNameFromDesc(desc.getName());
+String name = desc.getName();
 if (StringUtils.isEmpty(name)) {
 logger.info("Cube name should not be empty.");
 throw new BadRequestException("Cube name should not be empty.");

http://git-wip-us.apache.org/repos/asf/kylin/blob/7fe7fc3e/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 5c59e1a..85c9284 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -278,14 +278,6 @@ public class CubeService extends BasicService {
 accessService.clean(cube, true);
 }
 
-public static String getCubeNameFromDesc(String descName) {
-if (descName.toLowerCase().endsWith(DESC_SUFFIX)) {
-

[2/3] kylin git commit: KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager

2016-11-24 Thread liyang 
KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/917a8854
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/917a8854
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/917a8854

Branch: refs/heads/KYLIN-2217-2
Commit: 917a88543400d30d0ed3467167243104fc8fb70f
Parents: 7fe7fc3
Author: Li Yang 
Authored: Thu Nov 24 17:24:57 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 09:48:33 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 29 ++---
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  | 34 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |  2 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |  5 ++
 .../apache/kylin/dict/DictionaryManager.java| 65 +---
 .../engine/mr/steps/MergeCuboidMapperTest.java  |  2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |  6 +-
 7 files changed, 93 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/917a8854/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 307bb46..ea8ff81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -48,7 +48,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.DistinctColumnValuesProvider;
 import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
@@ -214,25 +213,39 @@ public class CubeManager implements IRealizationProvider {
 return result;
 }
 
-public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable) throws IOException {
 CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
 return null;
 
-DictionaryManager dictMgr = getDictionaryManager();
 String builderClass = cubeDesc.getDictionaryBuilderClass(col);
-DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), 
col, factTableValueProvider, builderClass);
+DictionaryInfo dictInfo = 
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, 
builderClass);
 
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable, Dictionary dict) throws IOException {
+CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
+return null;
+
+DictionaryInfo dictInfo = 
getDictionaryManager().saveDictionary(cubeDesc.getModel(), col, inpTable, dict);
+
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, 
DictionaryInfo dictInfo) throws IOException {
 if (dictInfo != null) {
 Dictionary dict = dictInfo.getDictionaryObject();
 cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
 cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), 
dict.getSize(), dict.getSizeOfId() });
 
-CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
-cubeBuilder.setToUpdateSegs(cubeSeg);
-updateCube(cubeBuilder);
+CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
+update.setToUpdateSegs(cubeSeg);
+updateCube(update);
 }
-return dictInfo;
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/kylin/blob/917a8854/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 89e2e9b..a6aeb96 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++

kylin git commit: KYLIN-2217 fix bug

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 e4a22531a -> d14de80ee


KYLIN-2217 fix bug


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d14de80e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d14de80e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d14de80e

Branch: refs/heads/KYLIN-2217-2
Commit: d14de80ee17691f21ab9225bd1f2880ea1b17c04
Parents: e4a2253
Author: Yang Li 
Authored: Fri Nov 25 07:16:45 2016 +0800
Committer: Yang Li 
Committed: Fri Nov 25 07:16:45 2016 +0800

--
 .../main/java/org/apache/kylin/engine/mr/DFSFileTable.java| 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d14de80e/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
index 8c1f6bd..6f337d0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
@@ -61,12 +61,13 @@ public class DFSFileTable implements ReadableTable {
 
 @Override
 public TableSignature getSignature() throws IOException {
+Pair sizeAndLastModified;
 try {
-Pair sizeAndLastModified = 
getSizeAndLastModified(path);
-return new TableSignature(path, sizeAndLastModified.getFirst(), 
sizeAndLastModified.getSecond());
+sizeAndLastModified = getSizeAndLastModified(path);
 } catch (FileNotFoundException ex) {
-return null;
+sizeAndLastModified = Pair.newPair(-1L, 0L);
 }
+return new TableSignature(path, sizeAndLastModified.getFirst(), 
sizeAndLastModified.getSecond());
 }
 
 @Override

[2/3] kylin git commit: minor, fix kylin-tools-log4j.properties location in pom.xml

2016-11-24 Thread liyang 
minor, fix kylin-tools-log4j.properties location in pom.xml


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6afcb269
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6afcb269
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6afcb269

Branch: refs/heads/KYLIN-2217-2
Commit: 6afcb2690c0d7eefd50e103e298aff80286776a4
Parents: e1d5b29
Author: Yang Li 
Authored: Thu Nov 24 21:55:19 2016 +0800
Committer: Yang Li 
Committed: Thu Nov 24 21:55:19 2016 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6afcb269/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 9628c0d..0501d0f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1097,7 +1097,7 @@
 
 
 log4j.configuration
-
file:${project.basedir}/build/conf/kylin-tools-log4j.properties
+
file:${pom.parent.basedir}/build/conf/kylin-tools-log4j.properties
 
 
 
-javaagent:${project.build.testOutputDirectory}/jamm.jar 
${argLine}

[3/3] kylin git commit: KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager

2016-11-24 Thread liyang 
KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e4a22531
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e4a22531
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e4a22531

Branch: refs/heads/KYLIN-2217-2
Commit: e4a22531a43cf6159a528d781e4b4fd37ea6261e
Parents: 6afcb26
Author: Li Yang 
Authored: Thu Nov 24 17:24:57 2016 +0800
Committer: Yang Li 
Committed: Thu Nov 24 21:56:35 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 47 +++---
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  | 34 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |  2 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |  5 ++
 .../apache/kylin/dict/DictionaryManager.java| 65 +---
 .../engine/mr/steps/MergeCuboidMapperTest.java  |  2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |  6 +-
 7 files changed, 93 insertions(+), 68 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e4a22531/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 9893040..3a4c754 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -48,7 +48,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.DistinctColumnValuesProvider;
 import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
@@ -214,25 +213,39 @@ public class CubeManager implements IRealizationProvider {
 return result;
 }
 
-public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable) throws IOException {
 CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
 return null;
 
-DictionaryManager dictMgr = getDictionaryManager();
 String builderClass = cubeDesc.getDictionaryBuilderClass(col);
-DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), 
col, factTableValueProvider, builderClass);
+DictionaryInfo dictInfo = 
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, 
builderClass);
 
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable, Dictionary dict) throws IOException {
+CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
+return null;
+
+DictionaryInfo dictInfo = 
getDictionaryManager().saveDictionary(cubeDesc.getModel(), col, inpTable, dict);
+
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, 
DictionaryInfo dictInfo) throws IOException {
 if (dictInfo != null) {
 Dictionary dict = dictInfo.getDictionaryObject();
 cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
 cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), 
dict.getSize(), dict.getSizeOfId() });
 
-CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
-cubeBuilder.setToUpdateSegs(cubeSeg);
-updateCube(cubeBuilder);
+CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
+update.setToUpdateSegs(cubeSeg);
+updateCube(update);
 }
-return dictInfo;
 }
 
 /**
@@ -617,24 +630,6 @@ public class CubeManager implements IRealizationProvider {
 }
 }
 
-private long calculateStartOffsetForAppendSegment(CubeInstance cube) {
-List existing = cube.getSegments();
-if (existing.isEmpty()) {
-return 0;
-} else {
-return existing.get(existing.size() - 1).getSourceOffsetEnd();
-}
-}
-
-private long calculateStartDateForAppendSegment(CubeInstance cube) {
-List existing = cube.getSegments();
-if

[1/3] kylin git commit: KYLIN-277 add API for pause and rollback job [Forced Update!]

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 fde30a157 -> e4a22531a (forced update)


KYLIN-277 add API for pause and rollback job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1d5b293
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1d5b293
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1d5b293

Branch: refs/heads/KYLIN-2217-2
Commit: e1d5b29385a5c044fe7f60d27d59baf82499b3de
Parents: 827205f
Author: shaofengshi 
Authored: Thu Nov 24 18:14:27 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 24 18:14:27 2016 +0800

--
 .../kylin/job/execution/AbstractExecutable.java |  2 +-
 .../kylin/job/execution/ExecutableManager.java  | 18 +
 .../kylin/job/execution/ExecutableState.java|  2 ++
 .../engine/mr/steps/SaveStatisticsStep.java |  1 -
 .../kylin/rest/controller/JobController.java| 21 +++-
 .../apache/kylin/rest/service/JobService.java   |  7 +++
 6 files changed, 48 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d5b293/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 551241b..cd9b033 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -82,7 +82,7 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 
 protected void onExecuteFinished(ExecuteResult result, ExecutableContext 
executableContext) {
 setEndTime(System.currentTimeMillis());
-if (!isDiscarded()) {
+if (!isDiscarded() && !isRunnable()) {
 if (result.succeed()) {
 getManager().updateJobOutput(getId(), ExecutableState.SUCCEED, 
null, result.output());
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d5b293/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 52d4d1c..4351e31 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -292,6 +292,24 @@ public class ExecutableManager {
 updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
 }
 
+
+public void rollbackJob(String jobId, String stepId) {
+AbstractExecutable job = getJob(jobId);
+if (job == null) {
+return;
+}
+
+if (job instanceof DefaultChainedExecutable) {
+List tasks = ((DefaultChainedExecutable) 
job).getTasks();
+for (AbstractExecutable task : tasks) {
+if (task.getId().compareTo(stepId) >= 0) {
+logger.debug("rollback task : " + task);
+updateJobOutput(task.getId(), ExecutableState.READY, null, 
null);
+}
+}
+}
+}
+
 public void pauseJob(String jobId) {
 AbstractExecutable job = getJob(jobId);
 if (job == null) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d5b293/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
index 0684eff..910bd7e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
@@ -69,6 +69,8 @@ public enum ExecutableState {
 VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, 
ExecutableState.STOPPED);
 
 
+//rollback
+VALID_STATE_TRANSFER.put(ExecutableState.SUCCEED, 
ExecutableState.READY);
 
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d5b293/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index

kylin git commit: minor, fix kylin-tools-log4j.properties location in pom.xml

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master e1d5b2938 -> 6afcb2690


minor, fix kylin-tools-log4j.properties location in pom.xml


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6afcb269
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6afcb269
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6afcb269

Branch: refs/heads/master
Commit: 6afcb2690c0d7eefd50e103e298aff80286776a4
Parents: e1d5b29
Author: Yang Li 
Authored: Thu Nov 24 21:55:19 2016 +0800
Committer: Yang Li 
Committed: Thu Nov 24 21:55:19 2016 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6afcb269/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 9628c0d..0501d0f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1097,7 +1097,7 @@
 
 
 log4j.configuration
-
file:${project.basedir}/build/conf/kylin-tools-log4j.properties
+
file:${pom.parent.basedir}/build/conf/kylin-tools-log4j.properties
 
 
 
-javaagent:${project.build.testOutputDirectory}/jamm.jar 
${argLine}

kylin git commit: KYLIN-277 add API for pause and rollback job

2016-11-24 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/master 827205f17 -> e1d5b2938


KYLIN-277 add API for pause and rollback job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1d5b293
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1d5b293
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1d5b293

Branch: refs/heads/master
Commit: e1d5b29385a5c044fe7f60d27d59baf82499b3de
Parents: 827205f
Author: shaofengshi 
Authored: Thu Nov 24 18:14:27 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 24 18:14:27 2016 +0800

--
 .../kylin/job/execution/AbstractExecutable.java |  2 +-
 .../kylin/job/execution/ExecutableManager.java  | 18 +
 .../kylin/job/execution/ExecutableState.java|  2 ++
 .../engine/mr/steps/SaveStatisticsStep.java |  1 -
 .../kylin/rest/controller/JobController.java| 21 +++-
 .../apache/kylin/rest/service/JobService.java   |  7 +++
 6 files changed, 48 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d5b293/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 551241b..cd9b033 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -82,7 +82,7 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 
 protected void onExecuteFinished(ExecuteResult result, ExecutableContext 
executableContext) {
 setEndTime(System.currentTimeMillis());
-if (!isDiscarded()) {
+if (!isDiscarded() && !isRunnable()) {
 if (result.succeed()) {
 getManager().updateJobOutput(getId(), ExecutableState.SUCCEED, 
null, result.output());
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d5b293/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 52d4d1c..4351e31 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -292,6 +292,24 @@ public class ExecutableManager {
 updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
 }
 
+
+public void rollbackJob(String jobId, String stepId) {
+AbstractExecutable job = getJob(jobId);
+if (job == null) {
+return;
+}
+
+if (job instanceof DefaultChainedExecutable) {
+List tasks = ((DefaultChainedExecutable) 
job).getTasks();
+for (AbstractExecutable task : tasks) {
+if (task.getId().compareTo(stepId) >= 0) {
+logger.debug("rollback task : " + task);
+updateJobOutput(task.getId(), ExecutableState.READY, null, 
null);
+}
+}
+}
+}
+
 public void pauseJob(String jobId) {
 AbstractExecutable job = getJob(jobId);
 if (job == null) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d5b293/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
index 0684eff..910bd7e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
@@ -69,6 +69,8 @@ public enum ExecutableState {
 VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, 
ExecutableState.STOPPED);
 
 
+//rollback
+VALID_STATE_TRANSFER.put(ExecutableState.SUCCEED, 
ExecutableState.READY);
 
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1d5b293/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 7718bfb..79346a5 100644
---

kylin git commit: KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager

2016-11-24 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2217-2 [created] fde30a157


KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fde30a15
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fde30a15
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fde30a15

Branch: refs/heads/KYLIN-2217-2
Commit: fde30a157dff088b796cc778be34201a4e6e6dd4
Parents: 827205f
Author: Li Yang 
Authored: Thu Nov 24 17:24:57 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 24 17:24:57 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 47 +++---
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  | 34 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |  2 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |  5 ++
 .../apache/kylin/dict/DictionaryManager.java| 65 +---
 .../engine/mr/steps/MergeCuboidMapperTest.java  |  2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |  6 +-
 7 files changed, 93 insertions(+), 68 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fde30a15/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 9893040..3a4c754 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -48,7 +48,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.DistinctColumnValuesProvider;
 import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
@@ -214,25 +213,39 @@ public class CubeManager implements IRealizationProvider {
 return result;
 }
 
-public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable) throws IOException {
 CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
 return null;
 
-DictionaryManager dictMgr = getDictionaryManager();
 String builderClass = cubeDesc.getDictionaryBuilderClass(col);
-DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), 
col, factTableValueProvider, builderClass);
+DictionaryInfo dictInfo = 
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, 
builderClass);
 
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable, Dictionary dict) throws IOException {
+CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
+return null;
+
+DictionaryInfo dictInfo = 
getDictionaryManager().saveDictionary(cubeDesc.getModel(), col, inpTable, dict);
+
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, 
DictionaryInfo dictInfo) throws IOException {
 if (dictInfo != null) {
 Dictionary dict = dictInfo.getDictionaryObject();
 cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
 cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), 
dict.getSize(), dict.getSizeOfId() });
 
-CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
-cubeBuilder.setToUpdateSegs(cubeSeg);
-updateCube(cubeBuilder);
+CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
+update.setToUpdateSegs(cubeSeg);
+updateCube(update);
 }
-return dictInfo;
 }
 
 /**
@@ -617,24 +630,6 @@ public class CubeManager implements IRealizationProvider {
 }
 }
 
-private long calculateStartOffsetForAppendSegment(CubeInstance cube) {
-List existing = cube.getSegments();
-if (existing.isEmpty()) {
-return 0;
-} else {
-return existing.get(existing.size() - 1).getSourceOffsetEnd();
-}
-}
-
-private long