[2/3] kylin git commit: KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager

2016-11-25 Thread liyang 
KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ff2d79c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ff2d79c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ff2d79c4

Branch: refs/heads/master
Commit: ff2d79c4e032ef243e584730738e5ba2d12ed5c0
Parents: 25f0819
Author: Li Yang 
Authored: Thu Nov 24 17:24:57 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 26 13:02:17 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 29 ++---
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  | 34 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |  2 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |  5 ++
 .../apache/kylin/dict/DictionaryManager.java| 65 +---
 .../engine/mr/steps/MergeCuboidMapperTest.java  |  2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |  6 +-
 7 files changed, 93 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ff2d79c4/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 307bb46..ea8ff81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -48,7 +48,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.DistinctColumnValuesProvider;
 import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
@@ -214,25 +213,39 @@ public class CubeManager implements IRealizationProvider {
 return result;
 }
 
-public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable) throws IOException {
 CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
 return null;
 
-DictionaryManager dictMgr = getDictionaryManager();
 String builderClass = cubeDesc.getDictionaryBuilderClass(col);
-DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), 
col, factTableValueProvider, builderClass);
+DictionaryInfo dictInfo = 
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, 
builderClass);
 
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable, Dictionary dict) throws IOException {
+CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
+return null;
+
+DictionaryInfo dictInfo = 
getDictionaryManager().saveDictionary(cubeDesc.getModel(), col, inpTable, dict);
+
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, 
DictionaryInfo dictInfo) throws IOException {
 if (dictInfo != null) {
 Dictionary dict = dictInfo.getDictionaryObject();
 cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
 cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), 
dict.getSize(), dict.getSizeOfId() });
 
-CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
-cubeBuilder.setToUpdateSegs(cubeSeg);
-updateCube(cubeBuilder);
+CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
+update.setToUpdateSegs(cubeSeg);
+updateCube(update);
 }
-return dictInfo;
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/kylin/blob/ff2d79c4/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 89e2e9b..a6aeb96 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++

[2/3] kylin git commit: KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager

2016-11-24 Thread liyang 
KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/917a8854
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/917a8854
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/917a8854

Branch: refs/heads/KYLIN-2217-2
Commit: 917a88543400d30d0ed3467167243104fc8fb70f
Parents: 7fe7fc3
Author: Li Yang 
Authored: Thu Nov 24 17:24:57 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 25 09:48:33 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 29 ++---
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  | 34 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |  2 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |  5 ++
 .../apache/kylin/dict/DictionaryManager.java| 65 +---
 .../engine/mr/steps/MergeCuboidMapperTest.java  |  2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |  6 +-
 7 files changed, 93 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/917a8854/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 307bb46..ea8ff81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -48,7 +48,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.DistinctColumnValuesProvider;
 import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
@@ -214,25 +213,39 @@ public class CubeManager implements IRealizationProvider {
 return result;
 }
 
-public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable) throws IOException {
 CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
 return null;
 
-DictionaryManager dictMgr = getDictionaryManager();
 String builderClass = cubeDesc.getDictionaryBuilderClass(col);
-DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), 
col, factTableValueProvider, builderClass);
+DictionaryInfo dictInfo = 
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, 
builderClass);
 
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable, Dictionary dict) throws IOException {
+CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
+return null;
+
+DictionaryInfo dictInfo = 
getDictionaryManager().saveDictionary(cubeDesc.getModel(), col, inpTable, dict);
+
+saveDictionaryInfo(cubeSeg, col, dictInfo);
+return dictInfo;
+}
+
+private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, 
DictionaryInfo dictInfo) throws IOException {
 if (dictInfo != null) {
 Dictionary dict = dictInfo.getDictionaryObject();
 cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
 cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), 
dict.getSize(), dict.getSizeOfId() });
 
-CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
-cubeBuilder.setToUpdateSegs(cubeSeg);
-updateCube(cubeBuilder);
+CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
+update.setToUpdateSegs(cubeSeg);
+updateCube(update);
 }
-return dictInfo;
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/kylin/blob/917a8854/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 89e2e9b..a6aeb96 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++