kylin git commit: Cleanup old storage after full regression test

2016-01-12 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 58c518188 -> ac045e155


Cleanup old storage after full regression test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac045e15
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac045e15
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac045e15

Branch: refs/heads/2.x-staging
Commit: ac045e155ef282e121f0d3112daab16bd250cc0e
Parents: 58c5181
Author: shaofengshi 
Authored: Wed Jan 13 10:09:05 2016 +0800
Committer: shaofengshi 
Committed: Wed Jan 13 10:34:01 2016 +0800

--
 .../org/apache/kylin/job/BuildCubeWithEngineTest.java | 11 ++-
 .../org/apache/kylin/job/BuildIIWithStreamTest.java   | 14 ++
 2 files changed, 24 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac045e15/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 53db99a..46bf1bb 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -33,6 +33,7 @@ import java.util.concurrent.Future;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractKylinTestCase;
 import org.apache.kylin.common.util.ClassUtil;
@@ -49,6 +50,7 @@ import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
 import org.apache.kylin.job.manager.ExecutableManager;
 import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
 import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
 import org.junit.After;
 import org.junit.Before;
@@ -103,9 +105,9 @@ public class BuildCubeWithEngineTest {
 
 @Before
 public void before() throws Exception {
-
 
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
 
+cleanupOldStorage();
 DeployUtil.initCliWorkDir();
 DeployUtil.deployMetadata();
 DeployUtil.overrideJobJarLocations();
@@ -322,4 +324,11 @@ public class BuildCubeWithEngineTest {
 return job.getId();
 }
 
+private int cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+
+int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+return exitCode;
+}
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac045e15/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index d4245cd..b420463 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -71,6 +71,8 @@ import 
org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.source.hive.HiveTableReader;
 import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -114,6 +116,11 @@ public class BuildIIWithStreamTest {
 }
 }
 }
+
+@AfterClass
+public void cleanup() throws Exception {
+cleanupOldStorage();
+}
 
 private String createIntermediateTable(IIDesc desc, KylinConfig 
kylinConfig) throws IOException {
 IIJoinedFlatTableDesc intermediateTableDesc = new 
IIJoinedFlatTableDesc(desc);
@@ -273,4 +280,11 @@ public class BuildIIWithStreamTest {
 return unsorted;
 }
 
+private int cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+
+int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+return exitCode;
+}
+
 }

kylin git commit: fix a compile error

2016-01-12 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging ac045e155 -> 05b7ca560


fix a compile error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/05b7ca56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/05b7ca56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/05b7ca56

Branch: refs/heads/2.x-staging
Commit: 05b7ca560c256deecbd95de52832bc3ad8691804
Parents: ac045e1
Author: shaofengshi 
Authored: Wed Jan 13 12:54:48 2016 +0800
Committer: shaofengshi 
Committed: Wed Jan 13 12:57:24 2016 +0800

--
 .../test/java/org/apache/kylin/job/BuildIIWithStreamTest.java| 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/05b7ca56/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index b420463..2317e62 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -118,7 +118,7 @@ public class BuildIIWithStreamTest {
 }
 
 @AfterClass
-public void cleanup() throws Exception {
+public static void cleanup() throws Exception {
 cleanupOldStorage();
 }
 
@@ -280,7 +280,7 @@ public class BuildIIWithStreamTest {
 return unsorted;
 }
 
-private int cleanupOldStorage() throws Exception {
+private static int cleanupOldStorage() throws Exception {
 String[] args = { "--delete", "true" };
 
 int exitCode = ToolRunner.run(new StorageCleanupJob(), args);

kylin git commit: KYLIN-1307 revisit growing dictionary

2016-01-12 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 3904a1767 -> 9f743fb78


KYLIN-1307 revisit growing dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9f743fb7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9f743fb7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9f743fb7

Branch: refs/heads/2.0-rc
Commit: 9f743fb7871d02f2c06e8fe21d288de1f502c939
Parents: 3904a17
Author: honma 
Authored: Tue Jan 12 16:06:39 2016 +0800
Committer: honma 
Committed: Tue Jan 12 16:06:39 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 +-
 .../apache/kylin/dict/DictionaryManager.java| 58 +++-
 .../apache/kylin/rest/service/CacheService.java |  2 +
 3 files changed, 51 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9f743fb7/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a5ca690..4c640f3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -412,7 +412,6 @@ public class KylinConfigBase implements Serializable {
 return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
 }
 
-
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }
@@ -453,6 +452,10 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
 }
 
+public boolean isGrowingDictEnabled() {
+return 
Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false"));
+}
+
 public int getHBaseScanMaxResultSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9f743fb7/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index b3cd634..cd12f01 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -139,26 +139,58 @@ public class DictionaryManager {
 
 initDictInfo(newDict, newDictInfo);
 
-DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
-if (largestDictInfo != null) {
-largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
-Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
-if (largestDictObject.contains(newDict)) {
-logger.info("dictionary content " + newDict + ", is contained 
by  dictionary at " + largestDictInfo.getResourcePath());
-return largestDictInfo;
-} else if (newDict.contains(largestDictObject)) {
-logger.info("dictionary content " + newDict + " is by far the 
largest, save it");
-return saveNewDict(newDictInfo);
+if (KylinConfig.getInstanceFromEnv().isGrowingDictEnabled()) {
+DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
+if (largestDictInfo != null) {
+largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
+Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
+if (largestDictObject.contains(newDict)) {
+logger.info("dictionary content " + newDict + ", is 
contained by  dictionary at " + largestDictInfo.getResourcePath());
+return largestDictInfo;
+} else if (newDict.contains(largestDictObject)) {
+logger.info("dictionary content " + newDict + " is by far 
the largest, save it");
+return saveNewDict(newDictInfo);
+} else {
+logger.info("merge dict and save...");
+return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+}
 } else {
-logger.info("merge dict and save...");
-return mergeDictionary(Lists.newArrayList(newDictInfo,

kylin git commit: KYLIN-1306 Null strings are not applied during fast cubing

2016-01-12 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5db430706 -> 829f5d312


KYLIN-1306 Null strings are not applied during fast cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/829f5d31
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/829f5d31
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/829f5d31

Branch: refs/heads/2.x-staging
Commit: 829f5d3129d0521d22d42bd43ba07a83bc2945be
Parents: 5db4307
Author: shaofengshi 
Authored: Tue Jan 12 16:23:04 2016 +0800
Committer: shaofengshi 
Committed: Tue Jan 12 17:28:34 2016 +0800

--
 .../InMemCubeBuilderInputConverter.java | 30 
 1 file changed, 30 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/829f5d31/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
index 3e9a67d..a1eb6da 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -17,10 +17,13 @@
 */
 package org.apache.kylin.cube.inmemcubing;
 
+import org.apache.kylin.common.util.Bytes;
+
 import java.util.List;
 import java.util.Map;
 
 import org.apache.kylin.common.util.Dictionary;
+import com.google.common.collect.Lists;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
 import org.apache.kylin.gridtable.GTInfo;
@@ -35,12 +38,15 @@ import org.apache.kylin.metadata.model.TblColRef;
  */
 public class InMemCubeBuilderInputConverter {
 
+public static final byte[] HIVE_NULL = Bytes.toBytes("\\N");
+
 private final CubeJoinedFlatTableDesc intermediateTableDesc;
 private final MeasureDesc[] measureDescs;
 private final MeasureIngester[] measureIngesters;
 private final int measureCount;
 private final Map dictionaryMap;
 private final GTInfo gtInfo;
+protected List nullBytes;
 
 
 public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map dictionaryMap, GTInfo gtInfo) {
@@ -50,6 +56,7 @@ public class InMemCubeBuilderInputConverter {
 this.measureDescs = cubeDesc.getMeasures().toArray(new 
MeasureDesc[measureCount]);
 this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures());
 this.dictionaryMap = dictionaryMap;
+initNullBytes(cubeDesc);
 }
 
 public final GTRecord convert(List row) {
@@ -73,6 +80,9 @@ public class InMemCubeBuilderInputConverter {
 
 for (int i = 0; i < keySize; i++) {
 key[i] = 
row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]);
+if (key[i] != null && isNull(Bytes.toBytes((String)key[i]))) {
+key[i] = null;
+}
 }
 
 return key;
@@ -112,4 +122,24 @@ public class InMemCubeBuilderInputConverter {
 return measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, 
dictionaryMap);
 }
 
+private void initNullBytes(CubeDesc cubeDesc) {
+nullBytes = Lists.newArrayList();
+nullBytes.add(HIVE_NULL);
+String[] nullStrings = cubeDesc.getNullStrings();
+if (nullStrings != null) {
+for (String s : nullStrings) {
+nullBytes.add(Bytes.toBytes(s));
+}
+}
+}
+
+private boolean isNull(byte[] v) {
+for (byte[] nullByte : nullBytes) {
+if (Bytes.equals(v, nullByte))
+return true;
+}
+return false;
+}
+
+
 }

kylin git commit: KYLIN-1251 enable nav tree expand and hide funciton

2016-01-12 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc b0ecfe750 -> d7eccda44


KYLIN-1251 enable nav tree expand and hide funciton


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7eccda4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7eccda4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7eccda4

Branch: refs/heads/2.0-rc
Commit: d7eccda448f380328ff740679537a5e74fda6f4a
Parents: b0ecfe7
Author: jian 
Authored: Tue Jan 12 17:39:16 2016 +0800
Committer: jian 
Committed: Tue Jan 12 17:42:58 2016 +0800

--
 pom.xml |  2 +-
 .../js/directives/kylin_abn_tree_directive.js   | 34 
 2 files changed, 21 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d7eccda4/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 0a7a764..325b58a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -917,8 +917,8 @@
 dictionary/metastore_db/**
 
 
-
webapp/app/css/AdminLTE-fonts.css
 webapp/app/css/AdminLTE.css
+
webapp/app/js/directives/kylin_abn_tree_directive.js
 
 
 webapp/app/routes.json

http://git-wip-us.apache.org/repos/asf/kylin/blob/d7eccda4/webapp/app/js/directives/kylin_abn_tree_directive.js
--
diff --git a/webapp/app/js/directives/kylin_abn_tree_directive.js 
b/webapp/app/js/directives/kylin_abn_tree_directive.js
index b2b2bd0..a3023ab 100644
--- a/webapp/app/js/directives/kylin_abn_tree_directive.js
+++ b/webapp/app/js/directives/kylin_abn_tree_directive.js
@@ -1,19 +1,24 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * The MIT License (MIT)
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * Copyright (c) 2013 Nick Perkins
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Permission is hereby granted, free of charge, to any person obtaining a 
copy of
+ * this software and associated documentation files (the "Software"), to deal 
in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
copies of
+ * the Software, and to permit persons to whom the Software is furnished to do 
so,
+ *   subject to the following conditions:
+ *
+ *   The above copyright notice and this permission notice shall be included 
in all
+ * copies or substantial portions of the Software.
+ *
+ *   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS 
OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
 (function() {
@@ -26,7 +31,7 @@
 '$timeout', function($timeout) {
   return {
 restrict: 'E',
-template: "\n   {{ row.label }} 
\n",
+template: "\n   {{ 
row.label }} \n",
 replace: true,
 scope: {
   treeData: '=',
@@ -122,6 +127,7 @@
 }
   };
   scope.user_clicks_branch = function(branch) {
+branch.expanded = !branch.expanded;
 if (branch !== selected_branch) {
   return select_branch(branch);
 }

kylin git commit: KYLIN-1182 Update Partition on CubeDesc from DataModelDesc during upgrade

2016-01-12 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 6adfc5c7c -> 7bfbbe2ff


KYLIN-1182 Update Partition on CubeDesc from DataModelDesc during upgrade


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7bfbbe2f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7bfbbe2f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7bfbbe2f

Branch: refs/heads/2.0-rc
Commit: 7bfbbe2ff6e45f63b5d78ec7df182c02d9c92cba
Parents: 6adfc5c
Author: lidongsjtu 
Authored: Tue Jan 12 21:10:48 2016 +0800
Committer: lidongsjtu 
Committed: Tue Jan 12 21:10:52 2016 +0800

--
 .../cube/upgrade/v2/CubeMetadataUpgradeV2.java| 18 ++
 1 file changed, 18 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7bfbbe2f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
index 54e1ffb..c285362 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
@@ -40,6 +40,7 @@ import org.apache.kylin.metadata.model.DimensionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.project.ProjectManager;
 
@@ -109,6 +110,7 @@ public class CubeMetadataUpgradeV2 {
 List cubeDescs = cubeDescManager.listAllDesc();
 for (CubeDesc cubeDesc : cubeDescs) {
 if (ArrayUtils.isEmpty(models) || ArrayUtils.contains(models, 
cubeDesc.getModelName())) {
+upgradeCubeDesc(cubeDesc);
 upgradeDataModelDesc(cubeDesc);
 upgradeCubeDescSignature(cubeDesc);
 }
@@ -159,6 +161,22 @@ public class CubeMetadataUpgradeV2 {
 }
 }
 
+private void upgradeCubeDesc(CubeDesc cubeDesc) {
+try {
+DataModelDesc modelDesc = cubeDesc.getModel();
+PartitionDesc modelPartDesc = modelDesc.getPartitionDesc();
+if (cubeDesc.getPartitionDateStart() == 0 && 
modelPartDesc.getPartitionDateStart() != 0) {
+
cubeDesc.setPartitionDateStart(modelPartDesc.getPartitionDateStart());
+
+store.putResource(cubeDesc.getResourcePath(), cubeDesc, 
CubeDescManager.CUBE_DESC_SERIALIZER);
+updatedResources.add(modelDesc.getResourcePath());
+}
+} catch (Exception e) {
+e.printStackTrace();
+errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: 
" + e.getLocalizedMessage());
+}
+}
+
 private void upgradeDataModelDesc(CubeDesc cubeDesc) {
 boolean upgrade = false;
 DataModelDesc modelDesc = cubeDesc.getModel();

kylin git commit: Cleanup old storage after full regression test

2016-01-12 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 7bfbbe2ff -> e0ff2a5c4


Cleanup old storage after full regression test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e0ff2a5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e0ff2a5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e0ff2a5c

Branch: refs/heads/2.0-rc
Commit: e0ff2a5c428aa046d8dbe37090914e2d07077da3
Parents: 7bfbbe2
Author: shaofengshi 
Authored: Wed Jan 13 10:09:05 2016 +0800
Committer: shaofengshi 
Committed: Wed Jan 13 10:17:47 2016 +0800

--
 .../org/apache/kylin/job/BuildCubeWithEngineTest.java | 11 ++-
 .../org/apache/kylin/job/BuildIIWithStreamTest.java   | 14 ++
 2 files changed, 24 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e0ff2a5c/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 53db99a..46bf1bb 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -33,6 +33,7 @@ import java.util.concurrent.Future;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractKylinTestCase;
 import org.apache.kylin.common.util.ClassUtil;
@@ -49,6 +50,7 @@ import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
 import org.apache.kylin.job.manager.ExecutableManager;
 import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
 import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
 import org.junit.After;
 import org.junit.Before;
@@ -103,9 +105,9 @@ public class BuildCubeWithEngineTest {
 
 @Before
 public void before() throws Exception {
-
 
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
 
+cleanupOldStorage();
 DeployUtil.initCliWorkDir();
 DeployUtil.deployMetadata();
 DeployUtil.overrideJobJarLocations();
@@ -322,4 +324,11 @@ public class BuildCubeWithEngineTest {
 return job.getId();
 }
 
+private int cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+
+int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+return exitCode;
+}
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/e0ff2a5c/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index d4245cd..b420463 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -71,6 +71,8 @@ import 
org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.source.hive.HiveTableReader;
 import org.apache.kylin.storage.hbase.HBaseConnection;
 import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -114,6 +116,11 @@ public class BuildIIWithStreamTest {
 }
 }
 }
+
+@AfterClass
+public void cleanup() throws Exception {
+cleanupOldStorage();
+}
 
 private String createIntermediateTable(IIDesc desc, KylinConfig 
kylinConfig) throws IOException {
 IIJoinedFlatTableDesc intermediateTableDesc = new 
IIJoinedFlatTableDesc(desc);
@@ -273,4 +280,11 @@ public class BuildIIWithStreamTest {
 return unsorted;
 }
 
+private int cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+
+int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+return exitCode;
+}
+
 }