kylin git commit: KYLIN-1312 Enhance DeployCoprocessorCLI to support Cube names filter

2016-01-13 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging bc8ea7817 -> 33b4e9655


KYLIN-1312 Enhance DeployCoprocessorCLI to support Cube names filter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/33b4e965
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/33b4e965
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/33b4e965

Branch: refs/heads/2.x-staging
Commit: 33b4e96552e3c690a6fe5a210c4e6c860238bff7
Parents: bc8ea78
Author: lidongsjtu 
Authored: Wed Jan 13 13:30:42 2016 +0800
Committer: lidongsjtu 
Committed: Wed Jan 13 18:08:19 2016 +0800

--
 .../hbase/util/DeployCoprocessorCLI.java| 60 ++--
 1 file changed, 43 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/33b4e965/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 1cdb9e9..101abc2 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -79,13 +79,20 @@ public class DeployCoprocessorCLI {
 
 List tableNames = getHTableNames(kylinConfig);
 logger.info("Identify tables " + tableNames);
-
-if (args.length == 1) {
-logger.info("Probe run, existing. Append argument 'all' or 
specific tables to execute.");
-System.exit(0);
+
+if (args.length <= 1) {
+printUsageAndExit();
 }
-
-tableNames = filterTables(tableNames, Arrays.asList(args).subList(1, 
args.length));
+
+String filterType = args[1].toLowerCase();
+if (filterType.equals("-table")) {
+tableNames = filterByTables(tableNames, 
Arrays.asList(args).subList(2, args.length));
+} else if (filterType.equals("-cube")) {
+tableNames = filterByCubes(tableNames, 
Arrays.asList(args).subList(2, args.length));
+} else if (!filterType.equals("all")) {
+printUsageAndExit();
+}
+
 logger.info("Will execute tables " + tableNames);
 
 Set oldJarPaths = getCoprocessorJarPaths(hbaseAdmin, 
tableNames);
@@ -105,19 +112,38 @@ public class DeployCoprocessorCLI {
 logger.info("Active coprocessor jar: " + hdfsCoprocessorJar);
 }
 
-private static List filterTables(List tableNames, 
List list) {
+private static void printUsageAndExit() {
+logger.info("Probe run, exiting. Append argument 'all' or specific 
tables/cubes to execute.");
+System.exit(0);
+}
+
+private static List filterByCubes(List allTableNames, 
List cubeNames) {
+CubeManager cubeManager = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+List result = Lists.newArrayList();
+for (String c : cubeNames) {
+c = c.trim();
+if (c.endsWith(","))
+c = c.substring(0, c.length() - 1);
+
+CubeInstance cubeInstance = cubeManager.getCube(c);
+for (CubeSegment segment : cubeInstance.getSegments()) {
+String tableName = segment.getStorageLocationIdentifier();
+if (allTableNames.contains(tableName)) {
+result.add(tableName);
+}
+}
+}
+return result;
+}
+
+private static List filterByTables(List allTableNames, 
List tableNames) {
 List result = Lists.newArrayList();
-for (String t : list) {
+for (String t : tableNames) {
 t = t.trim();
 if (t.endsWith(","))
 t = t.substring(0, t.length() - 1);
 
-if (t.equals("all")) {
-result.addAll(tableNames);
-break;
-}
-
-if (tableNames.contains(t)) {
+if (allTableNames.contains(t)) {
 result.add(t);
 }
 }
@@ -312,9 +338,9 @@ public class DeployCoprocessorCLI {
 }
 
 String jarPath = valueMatcher.group(1).trim();
-if (StringUtils.isNotEmpty(jarPath)) {
-result.add(jarPath);
-}
+if (StringUtils.isNotEmpty(jarPath)) {
+result.add(jarPath);
+}
 }
 }

[6/9] kylin git commit: KYLIN-1307 revisit growing dictionary

2016-01-13 Thread mahongbin 
KYLIN-1307 revisit growing dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c5784b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c5784b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c5784b2

Branch: refs/heads/2.x-staging
Commit: 1c5784b28ff9bbeb4a1ee9d4db3b05461641165d
Parents: 142eaf4
Author: honma 
Authored: Tue Jan 12 16:06:39 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:52:57 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 +-
 .../apache/kylin/dict/DictionaryManager.java| 58 +++-
 .../apache/kylin/rest/service/CacheService.java |  2 +
 3 files changed, 51 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c5784b2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f6775e8..bfad306 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -412,7 +412,6 @@ public class KylinConfigBase implements Serializable {
 return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
 }
 
-
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }
@@ -453,6 +452,10 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
 }
 
+public boolean isGrowingDictEnabled() {
+return 
Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false"));
+}
+
 public int getHBaseScanMaxResultSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c5784b2/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 630b2b2..d49e43d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -140,26 +140,58 @@ public class DictionaryManager {
 
 initDictInfo(newDict, newDictInfo);
 
-DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
-if (largestDictInfo != null) {
-largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
-Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
-if (largestDictObject.contains(newDict)) {
-logger.info("dictionary content " + newDict + ", is contained 
by  dictionary at " + largestDictInfo.getResourcePath());
-return largestDictInfo;
-} else if (newDict.contains(largestDictObject)) {
-logger.info("dictionary content " + newDict + " is by far the 
largest, save it");
-return saveNewDict(newDictInfo);
+if (KylinConfig.getInstanceFromEnv().isGrowingDictEnabled()) {
+DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
+if (largestDictInfo != null) {
+largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
+Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
+if (largestDictObject.contains(newDict)) {
+logger.info("dictionary content " + newDict + ", is 
contained by  dictionary at " + largestDictInfo.getResourcePath());
+return largestDictInfo;
+} else if (newDict.contains(largestDictObject)) {
+logger.info("dictionary content " + newDict + " is by far 
the largest, save it");
+return saveNewDict(newDictInfo);
+} else {
+logger.info("merge dict and save...");
+return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+}
 } else {
-logger.info("merge dict and save...");
-return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+logger.info("first dict of this column, save it

[9/9] kylin git commit: KYLIN-1308 maintain CubeHBaseScanRPC

2016-01-13 Thread mahongbin 
KYLIN-1308 maintain CubeHBaseScanRPC


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bc8ea781
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bc8ea781
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bc8ea781

Branch: refs/heads/2.x-staging
Commit: bc8ea78171856e6c438b1a5da5ed9e2b1a49a862
Parents: fe19e3d
Author: honma 
Authored: Wed Jan 13 17:42:54 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:54:04 2016 +0800

--
 .../apache/kylin/cube/kv/LazyRowKeyEncoder.java |  26 ++
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |  43 +++
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 348 +++
 .../hbase/cube/v2/CubeSegmentScanner.java   |   2 +-
 4 files changed, 266 insertions(+), 153 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bc8ea781/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java
index 0fca726..c93f65e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java
@@ -18,9 +18,15 @@
 
 package org.apache.kylin.cube.kv;
 
+import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.ShardingHash;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * A LazyRowKeyEncoder will not try to calculate shard
  * It works for both enableSharding or non-enableSharding scenario
@@ -38,4 +44,24 @@ public class LazyRowKeyEncoder extends RowKeyEncoder {
 throw new RuntimeException("If enableSharding false, you should 
never calculate shard");
 }
 }
+
+
+//for non-sharding cases it will only return one byte[] with not shard at 
beginning
+public List getRowKeysDifferentShards(byte[] halfCookedKey) {
+final short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId());
+
+if (!enableSharding) {
+return Lists.newArrayList(halfCookedKey);//not shard to append at 
head, so it is already well cooked
+} else {
+List ret = Lists.newArrayList();
+for (short i = 0; i < cuboidShardNum; ++i) {
+short shard = 
ShardingHash.normalize(cubeSeg.getCuboidBaseShard(cuboid.getId()), i, 
cubeSeg.getTotalShards());
+byte[] cookedKey = Arrays.copyOf(halfCookedKey, 
halfCookedKey.length);
+BytesUtil.writeShort(shard, cookedKey, 0, 
RowConstants.ROWKEY_SHARDID_LEN);
+ret.add(cookedKey);
+}
+return ret;
+}
+
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc8ea781/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 07143b1..db39455 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -30,6 +30,7 @@ import org.apache.kylin.gridtable.IGTScanner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 
@@ -104,6 +105,48 @@ public abstract class CubeHBaseRPC {
 return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, 
hbaseCaching, hbaseMaxResultSize);
 }
 
+protected List preparedHBaseScans(GTRecord pkStart, GTRecord 
pkEnd, List fuzzyKeys, ImmutableBitSet selectedColBlocks) {
+final List> selectedColumns = 
makeHBaseColumns(selectedColBlocks);
+List ret = Lists.newArrayList();
+
+LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid);
+byte[] start = encoder.createBuf();
+byte[] end = encoder.createBuf();
+List startKeys;
+List endKeys;
+
+encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);
+encoder.encode(pkStart, pkStart.getInfo().getPrimaryKey(), start);
+startKeys = encoder.getRowKeysDifferentShards(start);
+
+encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);
+encoder.encode(pkEnd,

[1/9] kylin git commit: KYLIN-1297 correct LoadingCache settings

2016-01-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 05b7ca560 -> bc8ea7817


KYLIN-1297 correct LoadingCache settings


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2d3d8597
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2d3d8597
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2d3d8597

Branch: refs/heads/2.x-staging
Commit: 2d3d859724f0f8fde2113ef187e14635e52dfd00
Parents: 05b7ca5
Author: honma 
Authored: Fri Jan 8 15:48:58 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:48:31 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java  | 5 +
 .../src/main/java/org/apache/kylin/dict/DictionaryManager.java  | 5 +++--
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2d3d8597/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 96c76f0..f6775e8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -408,6 +408,11 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }
 
+public int getCachedDictMaxEntrySize() {
+return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
+}
+
+
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2d3d8597/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index cb58721..cca8341 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -83,7 +83,8 @@ public class DictionaryManager {
 
 private DictionaryManager(KylinConfig config) {
 this.config = config;
-this.dictCache = 
CacheBuilder.newBuilder().weakValues().expireAfterWrite(10, 
TimeUnit.MINUTES).build(new CacheLoader() {
+this.dictCache = 
CacheBuilder.newBuilder().maximumSize(KylinConfig.getInstanceFromEnv().getCachedDictMaxEntrySize())//
+.expireAfterWrite(1, TimeUnit.DAYS).build(new 
CacheLoader() {
 @Override
 public DictionaryInfo load(String key) throws Exception {
 DictionaryInfo dictInfo = DictionaryManager.this.load(key, 
true);
@@ -357,7 +358,7 @@ public class DictionaryManager {
 DictionaryInfo load(String resourcePath, boolean loadDictObj) throws 
IOException {
 ResourceStore store = MetadataManager.getInstance(config).getStore();
 
-//logger.debug("Going to load DictionaryInfo from " + resourcePath);
+logger.debug("Loading DictionaryInfo(loadDictObj:" + loadDictObj + ") 
from " + resourcePath);
 DictionaryInfo info = store.getResource(resourcePath, 
DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : 
DictionaryInfoSerializer.INFO_SERIALIZER);
 
 //if (loadDictObj)

[5/9] kylin git commit: KYLIN-1301 TRUE/FALSE filter pruning

2016-01-13 Thread mahongbin 
KYLIN-1301 TRUE/FALSE filter pruning


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/142eaf41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/142eaf41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/142eaf41

Branch: refs/heads/2.x-staging
Commit: 142eaf41150a421d18f3fb584f58cff4df2d16da
Parents: a3a6a4d
Author: honma 
Authored: Wed Jan 13 17:50:41 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:51:17 2016 +0800

--
 .../kylin/gridtable/GTScanRangePlanner.java | 12 ++---
 .../kylin/gridtable/DictGridTableTest.java  | 21 +++
 query/src/test/resources/query/sql/query97.sql  | 28 
 3 files changed, 58 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/142eaf41/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 27fa6b4..6ff1ab4 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -18,6 +18,7 @@ import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.common.FuzzyValueCombination;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -49,7 +50,6 @@ public class GTScanRangePlanner {
  * @param partitionColRef the TblColRef in GT
  */
 public GTScanRangePlanner(GTInfo info, Pair 
segmentStartAndEnd, TblColRef partitionColRef) {
-  
 
 this.info = info;
 this.segmentStartAndEnd = segmentStartAndEnd;
@@ -188,7 +188,9 @@ public class GTScanRangePlanner {
 throw new IllegalStateException("Filter should be AND instead 
of " + andFilter);
 
 Collection andRanges = 
translateToAndDimRanges(andFilter.getChildren());
-result.add(andRanges);
+if (andRanges != null) {
+result.add(andRanges);
+}
 }
 
 return preEvaluateConstantConditions(result);
@@ -198,7 +200,11 @@ public class GTScanRangePlanner {
 Map rangeMap = new HashMap();
 for (TupleFilter filter : andFilters) {
 if ((filter instanceof CompareTupleFilter) == false) {
-continue;
+if (filter instanceof ConstantTupleFilter && 
!filter.evaluate(null, null)) {
+return null;
+} else {
+continue;
+}
 }
 
 CompareTupleFilter comp = (CompareTupleFilter) filter;

http://git-wip-us.apache.org/repos/asf/kylin/blob/142eaf41/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index 4588051..df69c17 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -89,6 +89,7 @@ public class DictGridTableTest {
 ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, 
"20"));
 ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, 
"30"));
 ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 
1, "30"));
+
 }
 
 @Test
@@ -138,6 +139,26 @@ public class DictGridTableTest {
 assertEquals("[142119360, null]-[null, null]", 
r.get(0).toString());
 assertEquals(0, r.get(0).fuzzyKeys.size());
 }
+{
+//skip FALSE filter
+LogicalTupleFilter filter = and(ageComp1, 
ConstantTupleFilter.FALSE);
+List r = planner.planScanRanges(filter);
+assertEquals(0, r.size());
+}
+{
+//TRUE or FALSE filter
+LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, 
ConstantTupleFilter.FALSE);
+List r = planner.planScanRanges(filter);
+assertEquals(1, r.size());
+assertEquals("[null, null]-[null, null]", r.get(0).toString());

kylin git commit: minor, UI 2.x advanced_setting display issue

2016-01-13 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc ae052bcc6 -> b54a423fc


minor, UI 2.x advanced_setting display issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b54a423f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b54a423f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b54a423f

Branch: refs/heads/2.0-rc
Commit: b54a423fcf6b30cc00aa9711f05539cddd07dab4
Parents: ae052bc
Author: jian 
Authored: Wed Jan 13 20:13:04 2016 +0800
Committer: jian 
Committed: Wed Jan 13 20:13:38 2016 +0800

--
 webapp/app/partials/cubeDesigner/advanced_settings.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b54a423f/webapp/app/partials/cubeDesigner/advanced_settings.html
--
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html 
b/webapp/app/partials/cubeDesigner/advanced_settings.html
index 87718df..eed5a26 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -44,7 +44,7 @@
 ng-options="rowkey_column.column as 
rowkey_column.column for rowkey_column in cubeMetaFrame.rowkey.rowkey_columns">
 
 
-{{aggregation_groups}}
+{{aggregation_groups}}

kylin git commit: KYLIN-1310 do not add default auto merge time when edit cube

2016-01-13 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc b54a423fc -> 5047e6013


KYLIN-1310 do not add default auto merge time when edit cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5047e601
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5047e601
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5047e601

Branch: refs/heads/2.0-rc
Commit: 5047e601395b6b7af3538c09b2660cd0ea8fe8ec
Parents: b54a423
Author: jian 
Authored: Wed Jan 13 20:32:15 2016 +0800
Committer: jian 
Committed: Wed Jan 13 20:32:15 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5047e601/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index fc02c6e..89712c9 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -32,7 +32,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 if($scope.state.mode==="edit") {
 $scope.metaModel = MetaModel;
   if(!$scope.cubeMetaFrame.auto_merge_time_ranges){
-$scope.cubeMetaFrame.auto_merge_time_ranges = [60480,241920];
+$scope.cubeMetaFrame.auto_merge_time_ranges = [];
   }
   $scope._auto_merge_time_ranges = [];
   
angular.forEach($scope.cubeMetaFrame.auto_merge_time_ranges,function(item){

[1/2] kylin git commit: minor, UI use select box input model name when edit cube

2016-01-13 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5fee6f0c8 -> 54e10692f


minor,UI use select box input model name when edit cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aca8d226
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aca8d226
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aca8d226

Branch: refs/heads/2.x-staging
Commit: aca8d22602987cf12e9df3218c11c25ac5dc8389
Parents: 5fee6f0
Author: jian 
Authored: Wed Jan 13 21:20:19 2016 +0800
Committer: jian 
Committed: Wed Jan 13 21:20:19 2016 +0800

--
 webapp/app/partials/cubeDesigner/info.html  | 14 ++
 webapp/app/partials/models/models_tree.html |  3 +++
 2 files changed, 13 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aca8d226/webapp/app/partials/cubeDesigner/info.html
--
diff --git a/webapp/app/partials/cubeDesigner/info.html 
b/webapp/app/partials/cubeDesigner/info.html
index 53279cf..14e319f 100644
--- a/webapp/app/partials/cubeDesigner/info.html
+++ b/webapp/app/partials/cubeDesigner/info.html
@@ -26,10 +26,16 @@
 Model Name
   
   
-
+
+   -- Choose Model -- 
+
+
 Model
 name is required.
 {{cubeMetaFrame.model_name}}
   

http://git-wip-us.apache.org/repos/asf/kylin/blob/aca8d226/webapp/app/partials/models/models_tree.html
--
diff --git a/webapp/app/partials/models/models_tree.html 
b/webapp/app/partials/models/models_tree.html
index 7b94841..03f5b07 100644
--- a/webapp/app/partials/models/models_tree.html
+++ b/webapp/app/partials/models/models_tree.html
@@ -61,3 +61,6 @@
   
 
 
+
+
+

[2/2] kylin git commit: minor, UI use select box input model name when edit cube disable box when edit

2016-01-13 Thread zhongjian 
minor,UI use select box input model name when edit cube disable box when edit


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/54e10692
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/54e10692
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/54e10692

Branch: refs/heads/2.x-staging
Commit: 54e10692faa19a977fa83614b673efb190799dae
Parents: aca8d22
Author: jian 
Authored: Wed Jan 13 21:20:41 2016 +0800
Committer: jian 
Committed: Wed Jan 13 21:20:41 2016 +0800

--
 webapp/app/partials/cubeDesigner/info.html | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/54e10692/webapp/app/partials/cubeDesigner/info.html
--
diff --git a/webapp/app/partials/cubeDesigner/info.html 
b/webapp/app/partials/cubeDesigner/info.html
index 14e319f..a2b2996 100644
--- a/webapp/app/partials/cubeDesigner/info.html
+++ b/webapp/app/partials/cubeDesigner/info.html
@@ -30,6 +30,7 @@
 ng-options="model for model in 
modelsManager.modelNameList"
 style="width:100%;"
 name="model_name"
+ng-disabled="{{isEdit}}"
 ng-required="true"
 data-placeholder="select a model"
 class="chosen-select">

[2/2] kylin git commit: minor, UI 2.x advanced_setting display issue

2016-01-13 Thread zhongjian 
minor, UI 2.x advanced_setting display issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/026a0c44
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/026a0c44
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/026a0c44

Branch: refs/heads/2.x-staging
Commit: 026a0c44a34861899049632221c5efbdc3931b70
Parents: 33b4e96
Author: jian 
Authored: Wed Jan 13 19:55:11 2016 +0800
Committer: jian 
Committed: Wed Jan 13 19:58:42 2016 +0800

--
 webapp/app/partials/cubeDesigner/advanced_settings.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/026a0c44/webapp/app/partials/cubeDesigner/advanced_settings.html
--
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html 
b/webapp/app/partials/cubeDesigner/advanced_settings.html
index 57abb75..cf9fb17 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -50,7 +50,7 @@
   
 
 
-
+
   {{aggregation_group.includes}}

[1/2] kylin git commit: minor, basic cuboid validation

2016-01-13 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 33b4e9655 -> 0d100c2ba


minor, basic cuboid validation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0d100c2b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0d100c2b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0d100c2b

Branch: refs/heads/2.x-staging
Commit: 0d100c2ba6d91a37c131455b699b00f5b38d3720
Parents: 026a0c4
Author: jian 
Authored: Wed Jan 13 19:56:09 2016 +0800
Committer: jian 
Committed: Wed Jan 13 19:58:42 2016 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0d100c2b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 5a28b00..c1212e2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -111,6 +111,9 @@ public class Cuboid implements Comparable {
 }
 
 public static long translateToValidCuboid(CubeDesc cubeDesc, long 
cuboidID) {
+if(cuboidID==getBaseCuboidId(cubeDesc)){
+return cuboidID;
+}
 List candidates = Lists.newArrayList();
 for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
 Long candidate = translateToValidCuboid(agg, cuboidID);

kylin git commit: minor, UI fix advanced_setting display issue

2016-01-13 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 6a9a30085 -> d1b607502


minor, UI fix advanced_setting display issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d1b60750
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d1b60750
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d1b60750

Branch: refs/heads/1.x-staging
Commit: d1b60750201e7a71b69ddf814987d7da7a5b4451
Parents: 6a9a300
Author: jian 
Authored: Wed Jan 13 20:39:50 2016 +0800
Committer: jian 
Committed: Wed Jan 13 20:39:50 2016 +0800

--
 webapp/app/partials/cubeDesigner/advanced_settings.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d1b60750/webapp/app/partials/cubeDesigner/advanced_settings.html
--
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html 
b/webapp/app/partials/cubeDesigner/advanced_settings.html
index 02a83fd..878e618 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -144,7 +144,7 @@
 ng-options="rowkey_column.column as 
rowkey_column.column for rowkey_column in cubeMetaFrame.rowkey.rowkey_columns">
 
 
-{{aggregation_groups}}
+{{aggregation_groups}}

kylin git commit: add reviewboardrc

2016-01-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 65566bf13 -> 5d2a58a1d


add reviewboardrc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d2a58a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d2a58a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d2a58a1

Branch: refs/heads/2.x-staging
Commit: 5d2a58a1d8dc8458c57072e97680aef0fe752cc9
Parents: 65566bf
Author: honma 
Authored: Thu Jan 14 10:25:05 2016 +0800
Committer: honma 
Committed: Thu Jan 14 10:38:00 2016 +0800

--
 .reviewboardrc | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5d2a58a1/.reviewboardrc
--
diff --git a/.reviewboardrc b/.reviewboardrc
new file mode 100644
index 000..7fcc51f
--- /dev/null
+++ b/.reviewboardrc
@@ -0,0 +1,4 @@
+REVIEWBOARD_URL = "https://reviews.apache.org;
+REPOSITORY = "kylin"
+BRANCH = "2.x-staging"
+LAND_DEST_BRANCH = "2.x-staging"

kylin git commit: KYLIN-1316 update wrong label in cube refresh page

2016-01-13 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging cef83f01c -> 5214b2b70


KYLIN-1316 update wrong label in cube refresh page


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5214b2b7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5214b2b7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5214b2b7

Branch: refs/heads/2.x-staging
Commit: 5214b2b70103d76355280deb7b05cc238a4d2799
Parents: cef83f0
Author: jian 
Authored: Thu Jan 14 11:01:41 2016 +0800
Committer: jian 
Committed: Thu Jan 14 11:01:41 2016 +0800

--
 ...pdate-wrong-label-in-cube-refresh-page.patch | 25 
 1 file changed, 25 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5214b2b7/0001-KYLIN-1316-update-wrong-label-in-cube-refresh-page.patch
--
diff --git a/0001-KYLIN-1316-update-wrong-label-in-cube-refresh-page.patch 
b/0001-KYLIN-1316-update-wrong-label-in-cube-refresh-page.patch
new file mode 100644
index 000..5da7612
--- /dev/null
+++ b/0001-KYLIN-1316-update-wrong-label-in-cube-refresh-page.patch
@@ -0,0 +1,25 @@
+From 7217298239ffa38252a30232fb11322810876f1b Mon Sep 17 00:00:00 2001
+From: jian 
+Date: Thu, 14 Jan 2016 10:56:36 +0800
+Subject: [PATCH] KYLIN-1316 update wrong label in cube refresh page
+
+---
+ webapp/app/partials/jobs/job_refresh.html | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/webapp/app/partials/jobs/job_refresh.html 
b/webapp/app/partials/jobs/job_refresh.html
+index 89705e0..b8ffa6e 100644
+--- a/webapp/app/partials/jobs/job_refresh.html
 b/webapp/app/partials/jobs/job_refresh.html
+@@ -35,7 +35,7 @@
+ 
{{metaModel.model.partition_desc.partition_date_column}}
+ 
+ 
+-MERGE START SEGMENT
++REFRESH SEGMENT
+ 
+

kylin git commit: KYLIN-1316 update wrong label in cube refresh page

2016-01-13 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc b5a444534 -> 64cb1f421


KYLIN-1316 update wrong label in cube refresh page


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64cb1f42
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64cb1f42
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64cb1f42

Branch: refs/heads/2.0-rc
Commit: 64cb1f421f61dcc67226d2f27bc45084f4ba63a8
Parents: b5a4445
Author: jian 
Authored: Thu Jan 14 11:02:33 2016 +0800
Committer: jian 
Committed: Thu Jan 14 11:02:56 2016 +0800

--
 webapp/app/partials/jobs/job_refresh.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/64cb1f42/webapp/app/partials/jobs/job_refresh.html
--
diff --git a/webapp/app/partials/jobs/job_refresh.html 
b/webapp/app/partials/jobs/job_refresh.html
index 89705e0..b8ffa6e 100644
--- a/webapp/app/partials/jobs/job_refresh.html
+++ b/webapp/app/partials/jobs/job_refresh.html
@@ -35,7 +35,7 @@
 
{{metaModel.model.partition_desc.partition_date_column}}
 
 
-MERGE START SEGMENT
+REFRESH SEGMENT

kylin git commit: add reviewboardrc

2016-01-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 721729823 -> bd73b643d


add reviewboardrc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd73b643
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd73b643
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd73b643

Branch: refs/heads/1.x-staging
Commit: bd73b643d307dcacf3e2d99e71f3b8f7008f1d1b
Parents: 7217298
Author: honma 
Authored: Thu Jan 14 10:59:03 2016 +0800
Committer: honma 
Committed: Thu Jan 14 10:59:07 2016 +0800

--
 .reviewboardrc | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bd73b643/.reviewboardrc
--
diff --git a/.reviewboardrc b/.reviewboardrc
new file mode 100644
index 000..79d75f5
--- /dev/null
+++ b/.reviewboardrc
@@ -0,0 +1,4 @@
+REVIEWBOARD_URL = "https://reviews.apache.org;
+REPOSITORY = "kylin"
+BRANCH = "1.x-staging"
+LAND_DEST_BRANCH = "1.x-staging"

kylin git commit: KYLIN-1221 Kill underlying running hadoop job while discard a job

2016-01-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5d2a58a1d -> cef83f01c


KYLIN-1221 Kill underlying running hadoop job while discard a job

Bugs closed: KYLIN-1221

Reviewed at https://reviews.apache.org/r/42285/


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cef83f01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cef83f01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cef83f01

Branch: refs/heads/2.x-staging
Commit: cef83f01c5f538cdacadbd5148f28c901148aa7d
Parents: 5d2a58a
Author: Dong Li 
Authored: Thu Jan 14 10:48:05 2016 +0800
Committer: honma 
Committed: Thu Jan 14 10:48:05 2016 +0800

--
 .../engine/mr/common/MapReduceExecutable.java   | 29 
 1 file changed, 18 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cef83f01/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index 585e0e1..f9c4673 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -53,12 +53,11 @@ import com.google.common.base.Preconditions;
  */
 public class MapReduceExecutable extends AbstractExecutable {
 
+public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
 private static final String KEY_MR_JOB = "MR_JOB_CLASS";
 private static final String KEY_PARAMS = "MR_JOB_PARAMS";
 private static final String KEY_COUNTER_SAVEAS = "MR_COUNTER_SAVEAS";
 
-public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
-
 public MapReduceExecutable() {
 super();
 }
@@ -117,7 +116,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 try {
 //for async mr job, ToolRunner just return 0;
 ToolRunner.run(hadoopJob, args);
-
+
 if (hadoopJob.isSkipped()) {
 return new ExecuteResult(ExecuteResult.State.SUCCEED, 
"skipped");
 }
@@ -169,7 +168,15 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 }
 
Thread.sleep(context.getConfig().getYarnStatusCheckIntervalSeconds() * 1000);
 }
-//TODO kill discarded mr job using "hadoop job -kill " + mrJobId
+
+// try to kill running map-reduce job to release resources.
+if (job != null) {
+try {
+job.killJob();
+} catch (Exception e) {
+logger.warn("failed to kill hadoop job: " + 
job.getJobID(), e);
+}
+}
 
 return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
 
@@ -211,7 +218,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 logger.info("kylin.job.yarn.app.rest.check.status.url" + " is not 
set, read from job configuration");
 }
 String rmWebHost = 
HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, 
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, job.getConfiguration());
-if(HAUtil.isHAEnabled(job.getConfiguration())) {
+if (HAUtil.isHAEnabled(job.getConfiguration())) {
 YarnConfiguration conf = new 
YarnConfiguration(job.getConfiguration());
 String active = RMHAUtils.findActiveRMHAId(conf);
 rmWebHost = 
HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS,
 active), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
@@ -236,22 +243,22 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 addExtraInfo(MAP_REDUCE_WAIT_TIME, t + "");
 }
 
-public void setMapReduceJobClass(Class 
clazzName) {
-setParam(KEY_MR_JOB, clazzName.getName());
-}
-
 public String getMapReduceJobClass() throws ExecuteException {
 return getParam(KEY_MR_JOB);
 }
 
-public void setMapReduceParams(String param) {
-setParam(KEY_PARAMS, param);
+public void setMapReduceJobClass(Class 
clazzName) {
+setParam(KEY_MR_JOB, clazzName.getName());
 }
 
 public String getMapReduceParams() {
 return getParam(KEY_PARAMS);
 }
 
+public void setMapReduceParams(String param) {
+setParam(KEY_PARAMS, param);
+}
+
 public String getCounterSaveAs() {
 return getParam(KEY_COUNTER_SAVEAS);
 }

kylin git commit: add reviewboardrc

2016-01-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 5feca4e7c -> b5a444534


add reviewboardrc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b5a44453
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b5a44453
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b5a44453

Branch: refs/heads/2.0-rc
Commit: b5a4445346bc576aa1b29e276e8a46112f3d54ce
Parents: 5feca4e
Author: honma 
Authored: Thu Jan 14 10:56:19 2016 +0800
Committer: honma 
Committed: Thu Jan 14 10:56:24 2016 +0800

--
 .reviewboardrc | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b5a44453/.reviewboardrc
--
diff --git a/.reviewboardrc b/.reviewboardrc
new file mode 100644
index 000..b6c24fa
--- /dev/null
+++ b/.reviewboardrc
@@ -0,0 +1,4 @@
+REVIEWBOARD_URL = "https://reviews.apache.org;
+REPOSITORY = "kylin"
+BRANCH = "2.0-rc"
+LAND_DEST_BRANCH = "2.0-rc"

kylin git commit: KYLIN-1221 Kill underlying running hadoop job while discard a job

2016-01-13 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc f10436c1c -> 5feca4e7c


KYLIN-1221 Kill underlying running hadoop job while discard a job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5feca4e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5feca4e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5feca4e7

Branch: refs/heads/2.0-rc
Commit: 5feca4e7cba15f793c9f99c4e0aefac1b72c7ad4
Parents: f10436c
Author: lidongsjtu 
Authored: Wed Jan 13 23:05:01 2016 +0800
Committer: lidongsjtu 
Committed: Wed Jan 13 23:12:37 2016 +0800

--
 .../engine/mr/common/MapReduceExecutable.java   | 29 
 1 file changed, 18 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5feca4e7/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index 585e0e1..f9c4673 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -53,12 +53,11 @@ import com.google.common.base.Preconditions;
  */
 public class MapReduceExecutable extends AbstractExecutable {
 
+public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
 private static final String KEY_MR_JOB = "MR_JOB_CLASS";
 private static final String KEY_PARAMS = "MR_JOB_PARAMS";
 private static final String KEY_COUNTER_SAVEAS = "MR_COUNTER_SAVEAS";
 
-public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
-
 public MapReduceExecutable() {
 super();
 }
@@ -117,7 +116,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 try {
 //for async mr job, ToolRunner just return 0;
 ToolRunner.run(hadoopJob, args);
-
+
 if (hadoopJob.isSkipped()) {
 return new ExecuteResult(ExecuteResult.State.SUCCEED, 
"skipped");
 }
@@ -169,7 +168,15 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 }
 
Thread.sleep(context.getConfig().getYarnStatusCheckIntervalSeconds() * 1000);
 }
-//TODO kill discarded mr job using "hadoop job -kill " + mrJobId
+
+// try to kill running map-reduce job to release resources.
+if (job != null) {
+try {
+job.killJob();
+} catch (Exception e) {
+logger.warn("failed to kill hadoop job: " + 
job.getJobID(), e);
+}
+}
 
 return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
 
@@ -211,7 +218,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 logger.info("kylin.job.yarn.app.rest.check.status.url" + " is not 
set, read from job configuration");
 }
 String rmWebHost = 
HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, 
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, job.getConfiguration());
-if(HAUtil.isHAEnabled(job.getConfiguration())) {
+if (HAUtil.isHAEnabled(job.getConfiguration())) {
 YarnConfiguration conf = new 
YarnConfiguration(job.getConfiguration());
 String active = RMHAUtils.findActiveRMHAId(conf);
 rmWebHost = 
HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS,
 active), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
@@ -236,22 +243,22 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 addExtraInfo(MAP_REDUCE_WAIT_TIME, t + "");
 }
 
-public void setMapReduceJobClass(Class 
clazzName) {
-setParam(KEY_MR_JOB, clazzName.getName());
-}
-
 public String getMapReduceJobClass() throws ExecuteException {
 return getParam(KEY_MR_JOB);
 }
 
-public void setMapReduceParams(String param) {
-setParam(KEY_PARAMS, param);
+public void setMapReduceJobClass(Class 
clazzName) {
+setParam(KEY_MR_JOB, clazzName.getName());
 }
 
 public String getMapReduceParams() {
 return getParam(KEY_PARAMS);
 }
 
+public void setMapReduceParams(String param) {
+setParam(KEY_PARAMS, param);
+}
+
 public String getCounterSaveAs() {
 return getParam(KEY_COUNTER_SAVEAS);
 }